Adding log rotation

README.md

@@ -74,4 +74,5 @@ do this instead
 
 * [x] Fix permission errors around opening the app.log and rules.json.
 * [x] Make the flags (config, rules) required instead of optional.
-* [ ] Figure how to use logrotate (a linux utility)
+~~* [ ] Figure how to use logrotate (a linux utility)~~
+* [ ] Figure how to do log rotation as part of this app's function

app.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"net/http"
+	"time"
 )
 
 type Application struct {
@@ -26,5 +27,6 @@ func (app *Application) Setup(port string) *http.Server {
 	return &http.Server{
 		Addr:        fmt.Sprintf(":%s", port),
 		Handler:     app.Mux,
+		ReadTimeout: 2500 * time.Millisecond,
 	}
 }

conf.go

@@ -4,8 +4,18 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
+	"strings"
 )
 
+type parsedConf struct {
+	RulesFp      string `json:"rulesPath"`
+	LogFp        string `json:"logPath"`
+	Compression  bool   `json:"compress"`
+	SizeToRotate string `json:"sizeToRotate"`
+	Port         string `json:"port"`
+}
+
 type Config struct {
 	MappingFilePath string
 	MappingRules    ImportRulesMappings
@@ -62,3 +72,98 @@ func (c *Config) LoadMappingFile(fp string) error {
 	}
 	return nil
 }
+
+func getDefaults() (map[string]string, error) {
+	m := make(map[string]string)
+	confDir, err := os.UserConfigDir()
+	if err != nil {
+		return m, err
+	}
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return m, err
+	}
+	m["rulesFp"] = filepath.Join(confDir, "gocustomcurls", "rules.json")
+	m["confFp"] = filepath.Join(confDir, "gocustomcurls", "config.json")
+	m["logfp"] = filepath.Join(homeDir, ".gocustomurls", "logs", "app.log")
+	return m, nil
+}
+
+func generateDefaultConfigFile() (parsedConf, error) {
+	var p parsedConf
+	defaults, err := getDefaults()
+	if err != nil {
+		return p, err
+	}
+	parentDir := filepath.Dir(defaults["confFp"])
+	err = os.MkdirAll(parentDir, 0755)
+	if err != nil {
+		return p, err
+	}
+	f, err := os.OpenFile(defaults["confFp"], os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
+	if err != nil {
+		return p, err
+	}
+	defer f.Close()
+	p.RulesFp = defaults["rulesFp"]
+	p.LogFp = defaults["logFp"]
+	p.Port = "7070"
+	p.Compression = true
+	p.SizeToRotate = "5MiB"
+	jsonString, _ := json.Marshal(p)
+	err = os.WriteFile(defaults["confFp"], jsonString, 0666)
+	if err != nil {
+		return p, err
+	}
+	return p, nil
+}
+
+func checkIfSizeIsConfigured(fsize string) (bool, error) {
+	suffixes := []string{"KB", "MB", "GB"}
+	var found string
+	for _, suffix := range suffixes {
+		if strings.HasSuffix(fsize, suffix) {
+			found = suffix
+		}
+	}
+	if len(found) == 0 {
+		return false, fmt.Errorf("%s has the incorrect suffix, Please use one of this suffixes {\"K\", \"KB\",\"M\", \"MB\", \"G\", \"GB\"}", fsize)
+	}
+
+	return true, nil
+}
+
+// load the main config file
+func (c *Config) LoadMainConfigFile(fp string) (parsedConf, error) {
+	var conf parsedConf
+	var err error
+	ok := isFile(fp)
+	if !ok {
+		// generate config file
+		errorLog.Println("Warning, generating default config file")
+		conf, err = generateDefaultConfigFile()
+		if err != nil {
+			return conf, err
+		}
+		c.MappingFilePath = conf.RulesFp
+		return conf, nil
+	}
+	f, err := os.Open(fp)
+	if err != nil {
+		return conf, err
+	}
+	defer f.Close()
+
+	err = json.NewDecoder(f).Decode(&conf)
+	if err != nil {
+		return conf, err
+	}
+
+	_, err = checkIfSizeIsConfigured(conf.SizeToRotate)
+	if err != nil {
+		return conf, err
+	}
+
+	c.MappingFilePath = conf.RulesFp
+	return conf, nil
+}

logger.go

@@ -1,12 +1,18 @@
 package main
 
 import (
+	"bufio"
+	"compress/gzip"
 	"encoding/json"
+	"fmt"
+	"io"
 	"log"
+	"math"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -43,6 +49,7 @@ type LogFile struct {
 	handle   *os.File
 	logger   *log.Logger
 	path     string
+	fileLock sync.Mutex
 }
 
 type LogFileRec struct {
@@ -51,28 +58,176 @@ type LogFileRec struct {
 	Url    string `json:"url"`
 }
 
-func newFileLogger(path string) (*LogFile, error) {
+func (lf *LogFile) makeCopyTo(dst string) error {
+	var err error
+	r, err := os.Open(lf.path)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	w, err := os.OpenFile(dst, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if c := w.Close(); err == nil {
+			err = c
+		}
+	}()
+
+	_, err = io.Copy(w, r)
+	return err
+}
+
+func (lf *LogFile) truncate() error {
+	fd, err := os.OpenFile(lf.path, os.O_TRUNC, 0666)
+	if err != nil {
+		return fmt.Errorf("could not open file %q for truncation: %v", lf.path, err)
+	}
+	err = fd.Close()
+	if err != nil {
+		return fmt.Errorf("could not close file handler for %q after truncation: %v", lf.path, err)
+	}
+	return nil
+}
+
+func prettyByteSize(b int64) string {
+	bf := float64(b)
+	for _, unit := range []string{"", "K", "M", "G", "T", "P", "E", "Z"} {
+		if math.Abs(bf) < 1024.0 {
+			return fmt.Sprintf("%3.1f%sB", bf, unit)
+		}
+		bf /= 1024.0
+	}
+	return fmt.Sprintf("%.1fYB", bf)
+}
+
+func compressOldFile(fname string) error {
+	reader, err := os.Open(fname)
+	if err != nil {
+		return fmt.Errorf("compressOldFile: failed to open existing file %s: %w", fname, err)
+	}
+	defer reader.Close()
+
+	buffer := bufio.NewReader(reader)
+	fnameGz := fname + ".gz"
+	fw, err := os.OpenFile(fnameGz, os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		return fmt.Errorf("compressOldFile: failed to open new file %s: %w", fnameGz, err)
+	}
+	defer fw.Close()
+
+	zw, err := gzip.NewWriterLevel(fw, gzip.BestCompression)
+	if err != nil {
+		return fmt.Errorf("compressOldFile: failed to create gzip writer: %w", err)
+	}
+	defer zw.Close()
+
+	_, err = buffer.WriteTo(zw)
+	if err != nil {
+		_ = zw.Close()
+		_ = fw.Close()
+		return fmt.Errorf("compressOldFile: failed to write to gz file: %w", err)
+	}
+	_ = reader.Close()
+
+	err = os.Remove(fname)
+	if err != nil {
+		return fmt.Errorf("compressOldFile: failed to delete old file: %w", err)
+	}
+	return nil
+}
+
+func (lf *LogFile) rotate() error {
+	lf.fileLock.Lock()
+	defer lf.fileLock.Unlock()
+
+	prefix := fmt.Sprintf("%s.%s", lf.handle.Name(), time.Now().Format("2006-01-02"))
+
+	// close file to allow for read-only access
+	err := lf.handle.Close()
+	if err != nil {
+		return err
+	}
+
+	// make a copy of the old log file
+	err = lf.makeCopyTo(prefix)
+	if err != nil {
+		return err
+	}
+
+	// compress the new log file
+	err = compressOldFile(prefix)
+	if err != nil {
+		return err
+	}
+
+	// truncate the old log file
+	err = lf.truncate()
+	if err != nil {
+		return err
+	}
+
+	f, err := os.OpenFile(lf.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
+	if err != nil {
+		return err
+	}
+
+	lf.handle = f
+	return nil
+}
+
+func (lf *LogFile) open(maxSize string) error {
+	f, err := os.OpenFile(lf.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
+	if err != nil {
+		return err
+	}
+	finfo, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	curSize := prettyByteSize(finfo.Size())
+	if curSize > maxSize {
+		err = lf.rotate()
+		if err != nil {
+			return err
+		}
+	}
+	lf.handle = f
+	lf.logger = log.New(f, "", 0)
+	return nil
+}
+
+func newFileLogger(path string, maxSize string) (*LogFile, error) {
 	requestedFile := filepath.Clean(filepath.Join("/", path))
 	parentDir := filepath.Dir(requestedFile)
-	err := os.MkdirAll(parentDir, 0777)
+	err := os.MkdirAll(parentDir, 0755)
 	if err != nil {
 		return nil, err
 	}
-	f, err := os.OpenFile(requestedFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
+	lf := &LogFile{
-	if err != nil {
-		return nil, err
-	}
-	return &LogFile{
-		handle: f,
-		logger: log.New(f, "", 0),
 		path: path,
-	}, nil
+	}
+	err = lf.open(maxSize)
+	return lf, err
+	// f, err := os.OpenFile(requestedFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
+	// if err != nil {
+	// 	return nil, err
+	// }
+	// return &LogFile{
+	// 	handle: f,
+	// 	logger: log.New(f, "", 0),
+	// 	path:   path,
+	// }, nil
 }
 
 func (f *LogFile) Close() error {
 	if f == nil {
 		return nil
 	}
+	f.fileLock.Lock()
+	defer f.fileLock.Unlock()
 	err := f.handle.Close()
 	f.handle = nil
 	return err
@@ -140,6 +295,8 @@ func (f *LogFile) WriteLog(r *http.Request) error {
 	if f == nil {
 		return nil
 	}
+	f.fileLock.Lock()
+	defer f.fileLock.Unlock()
 	var rec = make(map[string]string)
 	rec["method"] = r.Method
 	rec["requestUri"] = r.RequestURI

main.go

@@ -9,7 +9,6 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
-	"path/filepath"
 	"strconv"
 	"syscall"
 	"time"
@@ -33,27 +32,27 @@ func flagsSet(flags *flag.FlagSet) map[string]bool {
 }
 
 // generateDefaults - generate the default values for the rules.json and log files
-func generateDefaults(rulesfp string, logfp string) (string, string, error) {
+// func generateDefaults(rulesfp string, logfp string) (string, string, error) {
-	var newlogfp, newrulesfp string
+// 	var newlogfp, newrulesfp string
-	var err error
+// 	var err error
-	newlogfp = logfp
+// 	newlogfp = logfp
-	newrulesfp = rulesfp
+// 	newrulesfp = rulesfp
-	if len(newrulesfp) == 0 {
+// 	if len(newrulesfp) == 0 {
-		dir, err := os.UserConfigDir()
+// 		dir, err := os.UserConfigDir()
-		if err != nil {
+// 		if err != nil {
-			return newrulesfp, newlogfp, err
+// 			return newrulesfp, newlogfp, err
-		}
+// 		}
-		newrulesfp = filepath.Join(dir, "gocustomcurls", "rules.json")
+// 		newrulesfp = filepath.Join(dir, "gocustomcurls", "rules.json")
-	}
+// 	}
-	if len(newlogfp) == 0 {
+// 	if len(newlogfp) == 0 {
-		dir, err := os.UserHomeDir()
+// 		dir, err := os.UserHomeDir()
-		if err != nil {
+// 		if err != nil {
-			return newrulesfp, newlogfp, err
+// 			return newrulesfp, newlogfp, err
-		}
+// 		}
-		newlogfp = filepath.Join(dir, ".gocustomurls", "logs", "app.log")
+// 		newlogfp = filepath.Join(dir, ".gocustomurls", "logs", "app.log")
-	}
+// 	}
-	return newrulesfp, newlogfp, err
+// 	return newrulesfp, newlogfp, err
-}
+// }
 
 // isValidPort returns true if the port is valid
 // following the RFC https://datatracker.ietf.org/doc/html/rfc6056#section-2.1
@@ -74,9 +73,11 @@ func main() {
 		flags.PrintDefaults()
 	}
 
-	portFlag := flags.String("port", "7070", "Optional. Default port is 7070. Port to listen to")
+	confFlag := flags.String("conf", "", "Required. Contains all the configurations options")
-	rulesFileFlag := flags.String("config", "", "Optional. Contains go-import mapping")
+
-	logFileFlag := flags.String("logfile", "", "Optional. Default log file")
+	// portFlag := flags.String("port", "7070", "Optional. Default port is 7070. Port to listen to")
+	// rulesFileFlag := flags.String("rules", "", "Optional. Contains go-import mapping")
+	// logFileFlag := flags.String("logfile", "", "Optional. Default log file")
 	flags.Parse(os.Args[1:])
 
 	if len(flags.Args()) > 1 {
@@ -87,14 +88,23 @@ func main() {
 
 	allSetFlags := flagsSet(flags)
 
-	var port string
+	if !allSetFlags["conf"] {
-	if allSetFlags["port"] {
+		errorLog.Println("Error: conf arguments must be set")
-		port = *portFlag
+		flags.Usage()
-	} else {
+		os.Exit(1)
-		port = "7070"
 	}
 
-	p, err := strconv.Atoi(port)
+	// TODO: Use only one flag conf with a conf file that
+	// contains the following configuration, port, logfile, rulesfile, sizeofRotation
+	conf := *confFlag
+	c := &Config{}
+	pConf, err := c.LoadMainConfigFile(conf)
+	if err != nil {
+		errorLog.Println(err)
+		os.Exit(1)
+	}
+
+	p, err := strconv.Atoi(pConf.Port)
 	if err != nil {
 		errorLog.Println(err)
 		os.Exit(1)
@@ -105,41 +115,52 @@ func main() {
 		os.Exit(1)
 	}
 
-	var rulesFile string
+	err = c.LoadMappingFile(pConf.RulesFp)
-
+	if err != nil {
-	if allSetFlags["config"] {
+		errorLog.Println(err)
-		rulesFile = *rulesFileFlag
+		os.Exit(1)
 	}
-
+	l, err := newFileLogger(pConf.LogFp, pConf.SizeToRotate)
-	var logFile string
-	if allSetFlags["logFile"] {
-		logFile = *logFileFlag
-	}
-
-	rFile, lFile, err := generateDefaults(logFile, rulesFile)
 	if err != nil {
 		errorLog.Println(err)
 		os.Exit(1)
 	}
 
-	// load rules mapping
+	// var rulesFile string
-	c := &Config{}
+
-	err = c.LoadMappingFile(rFile)
+	// if allSetFlags["config"] {
-	if err != nil {
+	// 	rulesFile = *rulesFileFlag
-		errorLog.Println(err)
+	// }
-		os.Exit(1)
+
-	}
+	// var logFile string
-	l, err := newFileLogger(lFile)
+	// if allSetFlags["logFile"] {
-	if err != nil {
+	// 	logFile = *logFileFlag
-		errorLog.Println(err)
+	// }
-		os.Exit(1)
+
-	}
+	// rFile, lFile, err := generateDefaults(logFile, rulesFile)
+	// if err != nil {
+	// 	errorLog.Println(err)
+	// 	os.Exit(1)
+	// }
+
+	// // load rules mapping
+	// c := &Config{}
+	// err = c.LoadMappingFile(rFile)
+	// if err != nil {
+	// 	errorLog.Println(err)
+	// 	os.Exit(1)
+	// }
+	// l, err := newFileLogger(lFile)
+	// if err != nil {
+	// 	errorLog.Println(err)
+	// 	os.Exit(1)
+	// }
 
 	app := &Application{
 		Config: c,
 		Log:    l,
 	}
-	srv := app.Setup(port)
+	srv := app.Setup(pConf.Port)
 
 	// For graceful shutdowns
 	go func() {