From 7c636c35fea01cd0d59af8054599879cc75a12be Mon Sep 17 00:00:00 2001 From: iratusmachina Date: Wed, 19 Jun 2024 16:16:13 -0400 Subject: [PATCH] Adding log rotation --- README.md | 3 +- app.go | 6 +- conf.go | 105 +++++++++++++++++++++++++++++++ logger.go | 183 ++++++++++++++++++++++++++++++++++++++++++++++++++---- main.go | 131 ++++++++++++++++++++++---------------- 5 files changed, 357 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 70bbd75..c35d74a 100644 --- a/README.md +++ b/README.md @@ -74,4 +74,5 @@ do this instead * [x] Fix permission errors around opening the app.log and rules.json. * [x] Make the flags (config, rules) required instead of optional. -* [ ] Figure how to use logrotate (a linux utility) \ No newline at end of file +~~* [ ] Figure how to use logrotate (a linux utility)~~ +* [ ] Figure how to do log rotation as part of this app's function diff --git a/app.go b/app.go index 5043e9d..714876d 100644 --- a/app.go +++ b/app.go @@ -3,6 +3,7 @@ package main import ( "fmt" "net/http" + "time" ) type Application struct { @@ -24,7 +25,8 @@ func (app *Application) routes() { func (app *Application) Setup(port string) *http.Server { app.routes() return &http.Server{ - Addr: fmt.Sprintf(":%s", port), - Handler: app.Mux, + Addr: fmt.Sprintf(":%s", port), + Handler: app.Mux, + ReadTimeout: 2500 * time.Millisecond, } } diff --git a/conf.go b/conf.go index 81973b8..65c0eea 100644 --- a/conf.go +++ b/conf.go @@ -4,8 +4,18 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" + "strings" ) +type parsedConf struct { + RulesFp string `json:"rulesPath"` + LogFp string `json:"logPath"` + Compression bool `json:"compress"` + SizeToRotate string `json:"sizeToRotate"` + Port string `json:"port"` +} + type Config struct { MappingFilePath string MappingRules ImportRulesMappings @@ -62,3 +72,98 @@ func (c *Config) LoadMappingFile(fp string) error { } return nil } + +func getDefaults() (map[string]string, error) { + m := make(map[string]string) + confDir, err := os.UserConfigDir() + if err != nil { + return m, err + } + homeDir, err := os.UserHomeDir() + if err != nil { + return m, err + } + m["rulesFp"] = filepath.Join(confDir, "gocustomcurls", "rules.json") + m["confFp"] = filepath.Join(confDir, "gocustomcurls", "config.json") + m["logfp"] = filepath.Join(homeDir, ".gocustomurls", "logs", "app.log") + return m, nil +} + +func generateDefaultConfigFile() (parsedConf, error) { + var p parsedConf + defaults, err := getDefaults() + if err != nil { + return p, err + } + parentDir := filepath.Dir(defaults["confFp"]) + err = os.MkdirAll(parentDir, 0755) + if err != nil { + return p, err + } + f, err := os.OpenFile(defaults["confFp"], os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return p, err + } + defer f.Close() + p.RulesFp = defaults["rulesFp"] + p.LogFp = defaults["logFp"] + p.Port = "7070" + p.Compression = true + p.SizeToRotate = "5MiB" + jsonString, _ := json.Marshal(p) + err = os.WriteFile(defaults["confFp"], jsonString, 0666) + if err != nil { + return p, err + } + return p, nil +} + +func checkIfSizeIsConfigured(fsize string) (bool, error) { + suffixes := []string{"KB", "MB", "GB"} + var found string + for _, suffix := range suffixes { + if strings.HasSuffix(fsize, suffix) { + found = suffix + } + } + if len(found) == 0 { + return false, fmt.Errorf("%s has the incorrect suffix, Please use one of this suffixes {\"K\", \"KB\",\"M\", \"MB\", \"G\", \"GB\"}", fsize) + } + + return true, nil +} + +// load the main config file +func (c *Config) LoadMainConfigFile(fp string) (parsedConf, error) { + var conf parsedConf + var err error + ok := isFile(fp) + if !ok { + // generate config file + errorLog.Println("Warning, generating default config file") + conf, err = generateDefaultConfigFile() + if err != nil { + return conf, err + } + c.MappingFilePath = conf.RulesFp + return conf, nil + } + f, err := os.Open(fp) + if err != nil { + return conf, err + } + defer f.Close() + + err = json.NewDecoder(f).Decode(&conf) + if err != nil { + return conf, err + } + + _, err = checkIfSizeIsConfigured(conf.SizeToRotate) + if err != nil { + return conf, err + } + + c.MappingFilePath = conf.RulesFp + return conf, nil +} diff --git a/logger.go b/logger.go index b28fed4..46dc2ee 100644 --- a/logger.go +++ b/logger.go @@ -1,12 +1,18 @@ package main import ( + "bufio" + "compress/gzip" "encoding/json" + "fmt" + "io" "log" + "math" "net/http" "os" "path/filepath" "strings" + "sync" "time" ) @@ -40,9 +46,10 @@ var ( ) type LogFile struct { - handle *os.File - logger *log.Logger - path string + handle *os.File + logger *log.Logger + path string + fileLock sync.Mutex } type LogFileRec struct { @@ -51,28 +58,176 @@ type LogFileRec struct { Url string `json:"url"` } -func newFileLogger(path string) (*LogFile, error) { +func (lf *LogFile) makeCopyTo(dst string) error { + var err error + r, err := os.Open(lf.path) + if err != nil { + return err + } + defer r.Close() + + w, err := os.OpenFile(dst, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return err + } + defer func() { + if c := w.Close(); err == nil { + err = c + } + }() + + _, err = io.Copy(w, r) + return err +} + +func (lf *LogFile) truncate() error { + fd, err := os.OpenFile(lf.path, os.O_TRUNC, 0666) + if err != nil { + return fmt.Errorf("could not open file %q for truncation: %v", lf.path, err) + } + err = fd.Close() + if err != nil { + return fmt.Errorf("could not close file handler for %q after truncation: %v", lf.path, err) + } + return nil +} + +func prettyByteSize(b int64) string { + bf := float64(b) + for _, unit := range []string{"", "K", "M", "G", "T", "P", "E", "Z"} { + if math.Abs(bf) < 1024.0 { + return fmt.Sprintf("%3.1f%sB", bf, unit) + } + bf /= 1024.0 + } + return fmt.Sprintf("%.1fYB", bf) +} + +func compressOldFile(fname string) error { + reader, err := os.Open(fname) + if err != nil { + return fmt.Errorf("compressOldFile: failed to open existing file %s: %w", fname, err) + } + defer reader.Close() + + buffer := bufio.NewReader(reader) + fnameGz := fname + ".gz" + fw, err := os.OpenFile(fnameGz, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return fmt.Errorf("compressOldFile: failed to open new file %s: %w", fnameGz, err) + } + defer fw.Close() + + zw, err := gzip.NewWriterLevel(fw, gzip.BestCompression) + if err != nil { + return fmt.Errorf("compressOldFile: failed to create gzip writer: %w", err) + } + defer zw.Close() + + _, err = buffer.WriteTo(zw) + if err != nil { + _ = zw.Close() + _ = fw.Close() + return fmt.Errorf("compressOldFile: failed to write to gz file: %w", err) + } + _ = reader.Close() + + err = os.Remove(fname) + if err != nil { + return fmt.Errorf("compressOldFile: failed to delete old file: %w", err) + } + return nil +} + +func (lf *LogFile) rotate() error { + lf.fileLock.Lock() + defer lf.fileLock.Unlock() + + prefix := fmt.Sprintf("%s.%s", lf.handle.Name(), time.Now().Format("2006-01-02")) + + // close file to allow for read-only access + err := lf.handle.Close() + if err != nil { + return err + } + + // make a copy of the old log file + err = lf.makeCopyTo(prefix) + if err != nil { + return err + } + + // compress the new log file + err = compressOldFile(prefix) + if err != nil { + return err + } + + // truncate the old log file + err = lf.truncate() + if err != nil { + return err + } + + f, err := os.OpenFile(lf.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return err + } + + lf.handle = f + return nil +} + +func (lf *LogFile) open(maxSize string) error { + f, err := os.OpenFile(lf.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return err + } + finfo, err := f.Stat() + if err != nil { + return err + } + curSize := prettyByteSize(finfo.Size()) + if curSize > maxSize { + err = lf.rotate() + if err != nil { + return err + } + } + lf.handle = f + lf.logger = log.New(f, "", 0) + return nil +} + +func newFileLogger(path string, maxSize string) (*LogFile, error) { requestedFile := filepath.Clean(filepath.Join("/", path)) parentDir := filepath.Dir(requestedFile) - err := os.MkdirAll(parentDir, 0777) + err := os.MkdirAll(parentDir, 0755) if err != nil { return nil, err } - f, err := os.OpenFile(requestedFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) - if err != nil { - return nil, err + lf := &LogFile{ + path: path, } - return &LogFile{ - handle: f, - logger: log.New(f, "", 0), - path: path, - }, nil + err = lf.open(maxSize) + return lf, err + // f, err := os.OpenFile(requestedFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) + // if err != nil { + // return nil, err + // } + // return &LogFile{ + // handle: f, + // logger: log.New(f, "", 0), + // path: path, + // }, nil } func (f *LogFile) Close() error { if f == nil { return nil } + f.fileLock.Lock() + defer f.fileLock.Unlock() err := f.handle.Close() f.handle = nil return err @@ -140,6 +295,8 @@ func (f *LogFile) WriteLog(r *http.Request) error { if f == nil { return nil } + f.fileLock.Lock() + defer f.fileLock.Unlock() var rec = make(map[string]string) rec["method"] = r.Method rec["requestUri"] = r.RequestURI diff --git a/main.go b/main.go index fdb2cc6..b6171d1 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,6 @@ import ( "net/http" "os" "os/signal" - "path/filepath" "strconv" "syscall" "time" @@ -33,27 +32,27 @@ func flagsSet(flags *flag.FlagSet) map[string]bool { } // generateDefaults - generate the default values for the rules.json and log files -func generateDefaults(rulesfp string, logfp string) (string, string, error) { - var newlogfp, newrulesfp string - var err error - newlogfp = logfp - newrulesfp = rulesfp - if len(newrulesfp) == 0 { - dir, err := os.UserConfigDir() - if err != nil { - return newrulesfp, newlogfp, err - } - newrulesfp = filepath.Join(dir, "gocustomcurls", "rules.json") - } - if len(newlogfp) == 0 { - dir, err := os.UserHomeDir() - if err != nil { - return newrulesfp, newlogfp, err - } - newlogfp = filepath.Join(dir, ".gocustomurls", "logs", "app.log") - } - return newrulesfp, newlogfp, err -} +// func generateDefaults(rulesfp string, logfp string) (string, string, error) { +// var newlogfp, newrulesfp string +// var err error +// newlogfp = logfp +// newrulesfp = rulesfp +// if len(newrulesfp) == 0 { +// dir, err := os.UserConfigDir() +// if err != nil { +// return newrulesfp, newlogfp, err +// } +// newrulesfp = filepath.Join(dir, "gocustomcurls", "rules.json") +// } +// if len(newlogfp) == 0 { +// dir, err := os.UserHomeDir() +// if err != nil { +// return newrulesfp, newlogfp, err +// } +// newlogfp = filepath.Join(dir, ".gocustomurls", "logs", "app.log") +// } +// return newrulesfp, newlogfp, err +// } // isValidPort returns true if the port is valid // following the RFC https://datatracker.ietf.org/doc/html/rfc6056#section-2.1 @@ -74,9 +73,11 @@ func main() { flags.PrintDefaults() } - portFlag := flags.String("port", "7070", "Optional. Default port is 7070. Port to listen to") - rulesFileFlag := flags.String("config", "", "Optional. Contains go-import mapping") - logFileFlag := flags.String("logfile", "", "Optional. Default log file") + confFlag := flags.String("conf", "", "Required. Contains all the configurations options") + + // portFlag := flags.String("port", "7070", "Optional. Default port is 7070. Port to listen to") + // rulesFileFlag := flags.String("rules", "", "Optional. Contains go-import mapping") + // logFileFlag := flags.String("logfile", "", "Optional. Default log file") flags.Parse(os.Args[1:]) if len(flags.Args()) > 1 { @@ -87,14 +88,23 @@ func main() { allSetFlags := flagsSet(flags) - var port string - if allSetFlags["port"] { - port = *portFlag - } else { - port = "7070" + if !allSetFlags["conf"] { + errorLog.Println("Error: conf arguments must be set") + flags.Usage() + os.Exit(1) } - p, err := strconv.Atoi(port) + // TODO: Use only one flag conf with a conf file that + // contains the following configuration, port, logfile, rulesfile, sizeofRotation + conf := *confFlag + c := &Config{} + pConf, err := c.LoadMainConfigFile(conf) + if err != nil { + errorLog.Println(err) + os.Exit(1) + } + + p, err := strconv.Atoi(pConf.Port) if err != nil { errorLog.Println(err) os.Exit(1) @@ -105,41 +115,52 @@ func main() { os.Exit(1) } - var rulesFile string - - if allSetFlags["config"] { - rulesFile = *rulesFileFlag + err = c.LoadMappingFile(pConf.RulesFp) + if err != nil { + errorLog.Println(err) + os.Exit(1) } - - var logFile string - if allSetFlags["logFile"] { - logFile = *logFileFlag - } - - rFile, lFile, err := generateDefaults(logFile, rulesFile) + l, err := newFileLogger(pConf.LogFp, pConf.SizeToRotate) if err != nil { errorLog.Println(err) os.Exit(1) } - // load rules mapping - c := &Config{} - err = c.LoadMappingFile(rFile) - if err != nil { - errorLog.Println(err) - os.Exit(1) - } - l, err := newFileLogger(lFile) - if err != nil { - errorLog.Println(err) - os.Exit(1) - } + // var rulesFile string + + // if allSetFlags["config"] { + // rulesFile = *rulesFileFlag + // } + + // var logFile string + // if allSetFlags["logFile"] { + // logFile = *logFileFlag + // } + + // rFile, lFile, err := generateDefaults(logFile, rulesFile) + // if err != nil { + // errorLog.Println(err) + // os.Exit(1) + // } + + // // load rules mapping + // c := &Config{} + // err = c.LoadMappingFile(rFile) + // if err != nil { + // errorLog.Println(err) + // os.Exit(1) + // } + // l, err := newFileLogger(lFile) + // if err != nil { + // errorLog.Println(err) + // os.Exit(1) + // } app := &Application{ Config: c, Log: l, } - srv := app.Setup(port) + srv := app.Setup(pConf.Port) // For graceful shutdowns go func() {