Merge pull request #19831 from cloudflare/optimize-gelf

GELF logger: Add gelf-compression-type and gelf-compression-level
Upstream-commit: 0f59b0b12c37b5a3b4ad1c8a0cb91c93ff4228a9
Component: engine

components/engine/daemon/logger/gelf/gelf.go

@@ -6,9 +6,12 @@ package gelf
 
 import (
 	"bytes"
+	"compress/flate"
+	"encoding/json"
 	"fmt"
 	"net"
 	"net/url"
+	"strconv"
 	"time"
 
 	"github.com/Graylog2/go-gelf/gelf"
@@ -24,7 +27,7 @@ type gelfLogger struct {
 	writer   *gelf.Writer
 	ctx      logger.Context
 	hostname string
-	extra    map[string]interface{}
+	rawExtra json.RawMessage
 }
 
 func init() {
@@ -81,17 +84,43 @@ func New(ctx logger.Context) (logger.Logger, error) {
 		extra[k] = v
 	}
 
+	rawExtra, err := json.Marshal(extra)
+	if err != nil {
+		return nil, err
+	}
+
 	// create new gelfWriter
 	gelfWriter, err := gelf.NewWriter(address)
 	if err != nil {
 		return nil, fmt.Errorf("gelf: cannot connect to GELF endpoint: %s %v", address, err)
 	}
 
+	if v, ok := ctx.Config["gelf-compression-type"]; ok {
+		switch v {
+		case "gzip":
+			gelfWriter.CompressionType = gelf.CompressGzip
+		case "zlib":
+			gelfWriter.CompressionType = gelf.CompressZlib
+		case "none":
+			gelfWriter.CompressionType = gelf.CompressNone
+		default:
+			return nil, fmt.Errorf("gelf: invalid compression type %q", v)
+		}
+	}
+
+	if v, ok := ctx.Config["gelf-compression-level"]; ok {
+		val, err := strconv.Atoi(v)
+		if err != nil {
+			return nil, fmt.Errorf("gelf: invalid compression level %s, err %v", v, err)
+		}
+		gelfWriter.CompressionLevel = val
+	}
+
 	return &gelfLogger{
 		writer:   gelfWriter,
 		ctx:      ctx,
 		hostname: hostname,
-		extra:    extra,
+		rawExtra: rawExtra,
 	}, nil
 }
 
@@ -107,7 +136,7 @@ func (s *gelfLogger) Log(msg *logger.Message) error {
 		Short:    string(msg.Line),
 		TimeUnix: float64(msg.Timestamp.UnixNano()/int64(time.Millisecond)) / 1000.0,
 		Level:    level,
-		Extra:    s.extra,
+		RawExtra: s.rawExtra,
 	}
 
 	if err := s.writer.WriteMessage(&m); err != nil {
@@ -127,15 +156,26 @@ func (s *gelfLogger) Name() string {
 // ValidateLogOpt looks for gelf specific log options gelf-address, &
 // gelf-tag.
 func ValidateLogOpt(cfg map[string]string) error {
-	for key := range cfg {
+	for key, val := range cfg {
 		switch key {
 		case "gelf-address":
 		case "gelf-tag":
 		case "tag":
 		case "labels":
 		case "env":
+		case "gelf-compression-level":
+			i, err := strconv.Atoi(val)
+			if err != nil || i < flate.DefaultCompression || i > flate.BestCompression {
+				return fmt.Errorf("unknown value %q for log opt %q for gelf log driver", val, key)
+			}
+		case "gelf-compression-type":
+			switch val {
+			case "gzip", "zlib", "none":
+			default:
+				return fmt.Errorf("unknown value %q for log opt %q for gelf log driver", val, key)
+			}
 		default:
-			return fmt.Errorf("unknown log opt '%s' for gelf log driver", key)
+			return fmt.Errorf("unknown log opt %q for gelf log driver", key)
 		}
 	}