gocustomurls/logger.go

package main

import (
	"bufio"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

// some headers not worth logging
var (
	hdrsToNotLog = []string{
		"Accept-Language",
		"Cache-Control",
		"Cf-Ray",
		"CF-Visitor",
		"CF-Connecting-IP",
		"Cdn-Loop",
		"Cookie",
		"Connection",
		"Dnt",
		"If-Modified-Since",
		"Sec-Fetch-Dest",
		"Sec-Ch-Ua-Mobile",
		// "Sec-Ch-Ua",
		"Sec-Ch-Ua-Platform",
		"Sec-Fetch-Site",
		"Sec-Fetch-Mode",
		"Sec-Fetch-User",
		"Upgrade-Insecure-Requests",
		"X-Request-Start",
		"X-Forwarded-For",
		"X-Forwarded-Proto",
		"X-Forwarded-Host",
	}
	hdrsToNotLogMap map[string]bool
)

type LogFile struct {
	handle      *os.File
	logger      *log.Logger
	path        string
	fileLock    sync.Mutex
	canCompress bool
	maxSize     ByteSize
	curSize     ByteSize
}

type LogFileRec struct {
	Method string `json:"method"`
	IpAddr string `json:"ipAddr"`
	Url    string `json:"url"`
}

func (lf *LogFile) makeCopyTo(dst string) error {
	var err error
	r, err := os.Open(lf.path)
	if err != nil {
		return err
	}
	defer r.Close()

	w, err := os.OpenFile(dst, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer func() {
		if c := w.Close(); err == nil {
			err = c
		}
	}()

	_, err = io.Copy(w, r)
	return err
}

func (lf *LogFile) truncate() error {
	fd, err := os.OpenFile(lf.path, os.O_TRUNC, 0666)
	if err != nil {
		return fmt.Errorf("could not open file %q for truncation: %v", lf.path, err)
	}
	err = fd.Close()
	if err != nil {
		return fmt.Errorf("could not close file handler for %q after truncation: %v", lf.path, err)
	}
	return nil
}

func compressOldFile(fname string) error {
	reader, err := os.Open(fname)
	if err != nil {
		return fmt.Errorf("compressOldFile: failed to open existing file %s: %w", fname, err)
	}
	defer reader.Close()

	buffer := bufio.NewReader(reader)
	fnameGz := fname + ".gz"
	fw, err := os.OpenFile(fnameGz, os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		return fmt.Errorf("compressOldFile: failed to open new file %s: %w", fnameGz, err)
	}
	defer fw.Close()

	zw, err := gzip.NewWriterLevel(fw, gzip.BestCompression)
	if err != nil {
		return fmt.Errorf("compressOldFile: failed to create gzip writer: %w", err)
	}
	defer zw.Close()

	_, err = buffer.WriteTo(zw)
	if err != nil {
		_ = zw.Close()
		_ = fw.Close()
		return fmt.Errorf("compressOldFile: failed to write to gz file: %w", err)
	}
	_ = reader.Close()

	err = os.Remove(fname)
	if err != nil {
		return fmt.Errorf("compressOldFile: failed to delete old file: %w", err)
	}
	return nil
}

func (lf *LogFile) rotate() error {

	// new file
	newFilePrefix := fmt.Sprintf("%s.%s", lf.handle.Name(), time.Now().Format("2006-01-02"))

	// close file to allow for read-only access
	err := lf.handle.Close()
	if err != nil {
		return err
	}

	// make a copy of the old log file
	err = lf.makeCopyTo(newFilePrefix)
	if err != nil {
		return err
	}

	// compress the new log file
	if lf.canCompress {
		err = compressOldFile(newFilePrefix)
		if err != nil {
			return err
		}
	}

	// truncate the old log file
	err = lf.truncate()
	if err != nil {
		return err
	}

	f, err := os.OpenFile(lf.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}

	lf.handle = f
	return nil
}

func (lf *LogFile) open() error {
	lf.fileLock.Lock()
	defer lf.fileLock.Unlock()
	f, err := os.OpenFile(lf.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	lf.handle = f
	finfo, err := f.Stat()
	if err != nil {
		return err
	}
	curSize := finfo.Size()
	if lf.maxSize.NumberRep != 0 && curSize >= lf.maxSize.NumberRep {
		err = lf.rotate()
		if err != nil {
			return err
		}
	}
	lf.logger = log.New(f, "", 0)
	finfo, err = lf.handle.Stat()
	if err != nil {
		return err
	}
	by := ByteSize{}
	by.parseFromNumber(finfo.Size())
	lf.curSize = by
	return nil
}

func newFileLogger(path string, maxSize string, canCompress bool) (*LogFile, error) {
	requestedFile := filepath.Clean(filepath.Join("/", path))
	parentDir := filepath.Dir(requestedFile)
	err := os.MkdirAll(parentDir, 0755)
	if err != nil {
		return nil, err
	}
	by := ByteSize{}
	err = by.parseFromString(maxSize)
	if err != nil {
		return nil, err
	}
	lf := &LogFile{
		path:        path,
		canCompress: canCompress,
		maxSize:     by,
	}
	err = lf.open()
	return lf, err
}

func (f *LogFile) Close() error {
	if f == nil {
		return nil
	}
	f.fileLock.Lock()
	defer f.fileLock.Unlock()
	err := f.handle.Close()
	f.handle = nil
	return err
}

func extractFirstFragment(header *http.Header, headerName string) string {
	s := header.Get(headerName)
	if len(strings.TrimSpace(s)) == 0 {
		return s
	}
	fragments := strings.Split(s, ",")
	return strings.TrimSpace(fragments[0])
}

// Get Ip Address of the client
func extractIpAddress(r *http.Request) string {
	var ipAddr string
	if r == nil {
		return ""
	}
	possibleIpHeaders := []string{"CF-Connecting-IP", "X-Real-Ip", "X-Forwarded-For"}
	for _, header := range possibleIpHeaders {
		ipAddr = extractFirstFragment(&r.Header, header)
		if len(strings.TrimSpace(ipAddr)) != 0 {
			return ipAddr
		}
	}
	// pull ip from Request.RemoteAddr
	if len(strings.TrimSpace(r.RemoteAddr)) != 0 {
		index := strings.LastIndex(r.RemoteAddr, ";")
		if index == -1 {
			return r.RemoteAddr
		}
		ipAddr = r.RemoteAddr[:index]
	}
	return ipAddr
}

func canSkipExtraHeaders(r *http.Request) bool {
	ref := r.Header.Get("Referer")
	if len(strings.TrimSpace(ref)) == 0 {
		return false
	}
	return strings.Contains(ref, r.Host)
}

func shouldLogHeader(s string) bool {
	if hdrsToNotLogMap == nil {
		hdrsToNotLogMap = map[string]bool{}
		for _, h := range hdrsToNotLog {
			h = strings.ToLower(h)
			hdrsToNotLogMap[h] = true
		}
	}
	s = strings.ToLower(s)
	return !hdrsToNotLogMap[s]
}

func getCurrentDate() string {
	dt := time.Now()
	return dt.Format(time.RFC3339Nano)
}

func (lf *LogFile) WriteLog(r *http.Request) error {
	if lf == nil {
		return nil
	}
	lf.fileLock.Lock()
	defer lf.fileLock.Unlock()

	var rec = make(map[string]string)
	rec["method"] = r.Method
	rec["requestUri"] = r.RequestURI
	rec["Host"] = r.Host
	rec["ipAddr"] = extractIpAddress(r)
	rec["requestDate"] = getCurrentDate()
	if !canSkipExtraHeaders(r) {
		for key, val := range r.Header {
			if shouldLogHeader(key) && len(val) > 0 {
				rec[key] = val[0]
			}
		}
	}
	b, err := json.Marshal(rec)
	if err != nil {
		return err
	}
	lf.logger.Println(string(b))

	finfo, err := lf.handle.Stat()
	if err != nil {
		return err
	}
	curSize := finfo.Size()

	if lf.maxSize.NumberRep != 0 && curSize > lf.maxSize.NumberRep {
		err = lf.rotate()
		if err != nil {
			return err
		}
	}
	finfo, err = lf.handle.Stat()
	if err != nil {
		return err
	}
	by := ByteSize{}
	by.parseFromNumber(finfo.Size())
	lf.curSize = by
	return nil
}