Merge pull request #93 from jkaninda/feature/automated-healthcheck

feat: add automated route health check
This commit is contained in:
2024-11-12 15:41:21 +01:00
committed by GitHub
22 changed files with 286 additions and 79 deletions

View File

@@ -58,7 +58,6 @@ It's designed to be straightforward and efficient, offering features, like:
- Limit HTTP methods allowed for a particular route.
### Todo:
- [ ] Load Balancing Healthcheck, disable unavailable servers
- [ ] Blocklist IP address middleware
- [ ] Distributed Rate Limiting for In-Memory client IP based across multiple instances using Redis

View File

@@ -33,7 +33,7 @@ metadata:
data:
goma.yml: |
# Goma Gateway configurations
version: 0.1.7
version: 1.0
gateway:
# Proxy write timeout
writeTimeout: 15
@@ -47,6 +47,7 @@ data:
sslKeyFile: ''#key.pem
# Proxy rate limit, it's In-Memory IP based
rateLimit: 0
logLevel: info # debug, trace, off
accessLog: "/dev/Stdout"
errorLog: "/dev/stderr"
## Enable, disable routes health check
@@ -78,10 +79,10 @@ data:
##### Define routes
routes:
# Example of a route | 1
- name: Public # Name is optional
- path: /
name: Public # Name is optional
# host Domain/host based request routing
host: "" # Host is optional
path: /public
hosts: [] # Hosts are optional
## Rewrite a request path
# e.g rewrite: /store to /
rewrite: /
@@ -92,8 +93,6 @@ data:
# [X-Forwarded-Host, X-Forwarded-For, Host, Scheme ]
# It will not match the backend route, by default, it's disabled
disableHeaderXForward: false
# Internal health check
healthCheck: '' #/internal/health/ready
# Route Cors, global cors will be overridden by route
cors:
# Route Origins Cors, route will override global cors origins
@@ -113,17 +112,27 @@ data:
middlewares:
- api-forbidden-paths
# Example of a route | 2
- name: Basic auth
path: /protected
- path: /protected
name: Basic auth
rewrite: /
destination: https://example.com
methods: []
destination: ''
backends:
- https://example.com
- https://example2.com
methods:
- GET
# Route healthcheck
healthCheck:
path: /health/live
interval: 30s
timeout: 10s
healthyStatuses:
- 200
- 404
cors: {}
middlewares:
- api-forbidden-paths
- basic-auth
#Defines proxy middlewares
# middleware name must be unique
middlewares:
@@ -176,6 +185,46 @@ data:
- /api-docs/*
- /internal/*
- /actuator/*
- name: oauth-google
type: oauth
paths:
- /protected
- /example-of-oauth
rule:
clientId: xxx
clientSecret: xxx
provider: google
endpoint:
userInfoUrl: ""
redirectUrl: http://localhost:8080/callback
redirectPath: ""
cookiePath: ""
scopes:
- https://www.googleapis.com/auth/userinfo.email
- https://www.googleapis.com/auth/userinfo.profile
state: randomStateString
jwtSecret: your-strong-jwt-secret | It's optional
- name: oauth-authentik
type: oauth
paths:
- /protected
- /example-of-oauth
rule:
clientId: xxx
clientSecret: xxx
provider: custom
endpoint:
authUrl: https://authentik.example.com/application/o/authorize/
tokenUrl: https://authentik.example.com/application/o/token/
userInfoUrl: https://authentik.example.com/application/o/userinfo/
redirectUrl: http://localhost:8080/callback
redirectPath: ""
cookiePath: ""
scopes:
- email
- openid
state: randomStateString
jwtSecret: your-strong-jwt-secret | It's optional
```
## 3. Create Kubernetes deployment

View File

@@ -20,8 +20,8 @@ gateway:
methods: []
healthCheck:
path: "/health/live"
interval: 30 # in Seconds
timeout: 10 # in Seconds
interval: 30s
timeout: 10s
healthyStatuses: [200,404] # Healthy statuses
```

View File

@@ -8,7 +8,9 @@ nav_order: 4
# Load Balancing
Goma Gateway supports rund robim load blancing
Goma Gateway supports round-robin algorithm load balancing.
It comes with an integrated load balancing backends healthcheck.
```yaml
version: 1.0
@@ -23,8 +25,8 @@ gateway:
methods: []
healthCheck:
path: "/"
interval: 0
timeout: 0
interval: 30s
timeout: 10s
healthyStatuses: [200,404]
## destination: will be override by backends
destination: ""

View File

@@ -28,3 +28,7 @@ GOMA_LOG_LEVEL=trace
### When using a configuration file
Edit the Goma settings and set `logLevel: trace`.
### Disable logging
To disable logs, you need to set `logLevel: off`, it will turn off logs

View File

@@ -96,8 +96,8 @@ data:
# Route healthcheck
healthCheck:
path: /health/live
interval: 30
timeout: 10
interval: 30s
timeout: 10s
healthyStatuses:
- 200
- 404

View File

@@ -13,7 +13,7 @@ gateway:
sslKeyFile: ''#key.pem
# Proxy rate limit, it's In-Memory IP based
rateLimit: 0
logLevel: info # debug, trace
logLevel: info # debug, trace, off
accessLog: "/dev/Stdout"
errorLog: "/dev/stderr"
## Enable, disable routes health check
@@ -90,8 +90,8 @@ gateway:
# Route healthcheck
healthCheck:
path: /health/live
interval: 30
timeout: 10
interval: 30s
timeout: 10s
healthyStatuses:
- 200
- 404

1
go.mod
View File

@@ -29,6 +29,7 @@ require (
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
google.golang.org/protobuf v1.34.2 // indirect

2
go.sum
View File

@@ -41,6 +41,8 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=

View File

@@ -157,6 +157,8 @@ func initConfig(configFile string) error {
Rewrite: "/",
HealthCheck: RouteHealthCheck{
Path: "/",
Interval: "30s",
Timeout: "10s",
HealthyStatuses: []int{200, 404},
},
Middlewares: []string{"api-forbidden-paths"},
@@ -204,8 +206,8 @@ func initConfig(configFile string) error {
HealthCheck: RouteHealthCheck{
Path: "/health/live",
HealthyStatuses: []int{200, 404},
Interval: 30,
Timeout: 10,
Interval: "30s",
Timeout: "10s",
},
},
},

View File

@@ -69,24 +69,20 @@ func (heathRoute HealthCheckRoute) HealthCheckHandler(w http.ResponseWriter, r *
wg := sync.WaitGroup{}
wg.Add(len(heathRoute.Routes))
var routes []HealthCheckRouteResponse
for _, route := range heathRoute.Routes {
for _, health := range healthCheckRoutes(heathRoute.Routes) {
go func() {
defer wg.Done()
if route.HealthCheck.Path != "" {
err := healthCheck(route.Destination+route.HealthCheck.Path, route.HealthCheck.HealthyStatuses)
err := health.Check()
if err != nil {
if heathRoute.DisableRouteHealthCheckError {
routes = append(routes, HealthCheckRouteResponse{Name: route.Name, Status: "unhealthy", Error: "Route healthcheck errors disabled"})
routes = append(routes, HealthCheckRouteResponse{Name: health.Name, Status: "unhealthy", Error: "Route healthcheck errors disabled"})
}
routes = append(routes, HealthCheckRouteResponse{Name: route.Name, Status: "unhealthy", Error: "Error: " + err.Error()})
routes = append(routes, HealthCheckRouteResponse{Name: health.Name, Status: "unhealthy", Error: "Error: " + err.Error()})
} else {
logger.Debug("Route %s is healthy", route.Name)
routes = append(routes, HealthCheckRouteResponse{Name: route.Name, Status: "healthy", Error: ""})
}
} else {
logger.Debug("Route %s's healthCheck is undefined", route.Name)
routes = append(routes, HealthCheckRouteResponse{Name: route.Name, Status: "undefined", Error: ""})
logger.Debug("Route %s is healthy", health.Name)
routes = append(routes, HealthCheckRouteResponse{Name: health.Name, Status: "healthy", Error: ""})
}
}()
}

View File

@@ -18,27 +18,29 @@ limitations under the License.
import (
"fmt"
"github.com/jkaninda/goma-gateway/pkg/logger"
"github.com/jkaninda/goma-gateway/util"
"github.com/robfig/cron/v3"
"io"
"net/http"
"net/url"
"slices"
)
func healthCheck(healthURL string, healthyStatuses []int) error {
healthCheckURL, err := url.Parse(healthURL)
func (health Health) Check() error {
healthCheckURL, err := url.Parse(health.URL)
if err != nil {
return fmt.Errorf("error parsing HealthCheck URL: %v ", err)
}
// Create a new request for the route
healthReq, err := http.NewRequest("GET", healthCheckURL.String(), nil)
if err != nil {
return fmt.Errorf("error creating HealthCheck request: %v ", err)
return fmt.Errorf("error route %s: creating HealthCheck request: %v ", health.Name, err)
}
// Perform the request to the route's healthcheck
client := &http.Client{}
client := &http.Client{Timeout: health.TimeOut}
healthResp, err := client.Do(healthReq)
if err != nil {
logger.Error("Error performing HealthCheck request: %v ", err)
logger.Debug("Error route %s: performing HealthCheck request: %v ", health.Name, err)
return fmt.Errorf("error performing HealthCheck request: %v ", err)
}
defer func(Body io.ReadCloser) {
@@ -46,16 +48,58 @@ func healthCheck(healthURL string, healthyStatuses []int) error {
if err != nil {
}
}(healthResp.Body)
if len(healthyStatuses) > 0 {
if !slices.Contains(healthyStatuses, healthResp.StatusCode) {
logger.Error("Error performing HealthCheck request: %v ", err)
return fmt.Errorf("health check failed with status code %v", healthResp.StatusCode)
if len(health.HealthyStatuses) > 0 {
if !slices.Contains(health.HealthyStatuses, healthResp.StatusCode) {
logger.Debug("Error: Route %s: health check failed with status code %d", health.Name, healthResp.StatusCode)
return fmt.Errorf("health check failed with status code %d", healthResp.StatusCode)
}
} else {
if healthResp.StatusCode >= 400 {
logger.Debug("Error performing HealthCheck request: %v ", err)
return fmt.Errorf("health check failed with status code %v", healthResp.StatusCode)
logger.Debug("Error: Route %s: health check failed with status code %d", health.Name, healthResp.StatusCode)
return fmt.Errorf("health check failed with status code %d", healthResp.StatusCode)
}
}
return nil
}
func routesHealthCheck(routes []Route) {
for _, health := range healthCheckRoutes(routes) {
go func() {
err := health.createHealthCheckJob()
if err != nil {
logger.Error("Error creating healthcheck job: %v ", err)
return
}
}()
}
}
func (health Health) createHealthCheckJob() error {
interval := "30s"
if len(health.Interval) > 0 {
interval = health.Interval
}
expression := fmt.Sprintf("@every %s", interval)
if !util.IsValidCronExpression(expression) {
logger.Error("Health check interval is invalid: %s", interval)
logger.Info("Route health check ignored")
return fmt.Errorf("health check interval is invalid: %s", interval)
}
// Create a new cron instance
c := cron.New()
_, err := c.AddFunc(expression, func() {
err := health.Check()
if err != nil {
logger.Error("Route %s is unhealthy: error %v", health.Name, err.Error())
return
}
logger.Info("Route %s is healthy", health.Name)
})
if err != nil {
return err
}
// Start the cron scheduler
c.Start()
defer c.Stop()
select {}
}

View File

@@ -17,6 +17,7 @@ import (
"github.com/golang-jwt/jwt"
"github.com/jedib0t/go-pretty/v6/table"
"github.com/jkaninda/goma-gateway/pkg/logger"
"github.com/jkaninda/goma-gateway/util"
"golang.org/x/oauth2"
"net/http"
"time"
@@ -100,3 +101,43 @@ func createJWT(email, jwtSecret string) (string, error) {
return signedToken, nil
}
// healthCheckRoutes creates []Health
func healthCheckRoutes(routes []Route) []Health {
var healthRoutes []Health
for _, route := range routes {
if len(route.HealthCheck.Path) > 0 {
timeout, _ := util.ParseDuration("")
if len(route.HealthCheck.Timeout) > 0 {
d1, err1 := util.ParseDuration(route.HealthCheck.Timeout)
if err1 != nil {
logger.Error("Health check timeout is invalid: %s", route.HealthCheck.Timeout)
}
timeout = d1
}
if len(route.Backends) > 0 {
for index, backend := range route.Backends {
health := Health{
Name: fmt.Sprintf("%s - [%d]", route.Name, index),
URL: backend + route.HealthCheck.Path,
TimeOut: timeout,
HealthyStatuses: route.HealthCheck.HealthyStatuses,
}
healthRoutes = append(healthRoutes, health)
}
} else {
health := Health{
Name: route.Name,
URL: route.Destination + route.HealthCheck.Path,
TimeOut: timeout,
HealthyStatuses: route.HealthCheck.HealthyStatuses,
}
healthRoutes = append(healthRoutes, health)
}
} else {
logger.Debug("Route %s's healthCheck is undefined", route.Name)
}
}
return healthRoutes
}

View File

@@ -72,12 +72,10 @@ func (proxyRoute ProxyRoute) ProxyHandler() http.HandlerFunc {
}
// Update the headers to allow for SSL redirection
if !proxyRoute.disableXForward {
r.URL.Host = targetURL.Host
r.URL.Scheme = targetURL.Scheme
r.Header.Set("X-Forwarded-Host", r.Header.Get("Host"))
r.Header.Set("X-Forwarded-For", getRealIP(r))
r.Header.Set("X-Real-IP", getRealIP(r))
r.Host = targetURL.Host
}
backendURL, _ := url.Parse(proxyRoute.destination)
if len(proxyRoute.backends) > 0 {

View File

@@ -35,6 +35,8 @@ func init() {
func (gatewayServer GatewayServer) Initialize() *mux.Router {
gateway := gatewayServer.gateway
middlewares := gatewayServer.middlewares
//Routes background healthcheck
routesHealthCheck(gateway.Routes)
r := mux.NewRouter()
heath := HealthCheckRoute{
DisableRouteHealthCheckError: gateway.DisableRouteHealthCheckError,

View File

@@ -26,6 +26,7 @@ import (
"time"
)
// Start starts the server
func (gatewayServer GatewayServer) Start(ctx context.Context) error {
logger.Info("Initializing routes...")
route := gatewayServer.Initialize()
@@ -64,7 +65,6 @@ func (gatewayServer GatewayServer) Start(ctx context.Context) error {
}
// Set KeepAlive
httpServer.SetKeepAlivesEnabled(!gatewayServer.gateway.DisableKeepAlive)
httpsServer.SetKeepAlivesEnabled(!gatewayServer.gateway.DisableKeepAlive)
go func() {
logger.Info("Starting HTTP server listen=0.0.0.0:8080")
if err := httpServer.ListenAndServe(); err != nil {

View File

@@ -3,7 +3,6 @@ package pkg
import (
"context"
"log"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
@@ -23,32 +22,31 @@ func TestInit(t *testing.T) {
func TestCheckConfig(t *testing.T) {
TestInit(t)
initConfig(configFile)
err := CheckConfig(configFile)
err := initConfig(configFile)
if err != nil {
t.Error(err)
t.Fatalf(err.Error())
}
err = CheckConfig(configFile)
if err != nil {
t.Fatalf(err.Error())
}
log.Println("Goma Gateway configuration file checked successfully")
}
func TestStart(t *testing.T) {
TestInit(t)
initConfig(configFile)
err := initConfig(configFile)
if err != nil {
t.Fatalf(err.Error())
}
g := GatewayServer{}
gatewayServer, err := g.Config(configFile)
if err != nil {
t.Error(err)
}
route := gatewayServer.Initialize()
route.HandleFunc("/test", func(rw http.ResponseWriter, r *http.Request) {
_, err := rw.Write([]byte("Hello Goma Proxy"))
if err != nil {
t.Fatalf("Failed writing HTTP response: %v", err)
}
})
assertResponseBody := func(t *testing.T, s *httptest.Server, expectedBody string) {
resp, err := s.Client().Get(s.URL)
assertResponseBody := func(t *testing.T, s *httptest.Server) {
resp, err := s.Client().Get(s.URL + "/health/live")
if err != nil {
t.Fatalf("unexpected error getting from server: %v", err)
}
@@ -68,7 +66,7 @@ func TestStart(t *testing.T) {
t.Run("httpServer", func(t *testing.T) {
s := httptest.NewServer(route)
defer s.Close()
assertResponseBody(t, s, "Hello Goma Proxy")
assertResponseBody(t, s)
})
ctx.Done()
}

View File

@@ -20,6 +20,7 @@ package pkg
import (
"context"
"github.com/gorilla/mux"
"time"
)
type Config struct {
@@ -146,11 +147,11 @@ type Route struct {
//
// Methods allowed method
Methods []string `yaml:"methods"`
// HealthCheck Defines the backend is health
HealthCheck RouteHealthCheck `yaml:"healthCheck"`
// Destination Defines backend URL
Destination string `yaml:"destination"`
Backends []string `yaml:"backends"`
// HealthCheck Defines the backend is health
HealthCheck RouteHealthCheck `yaml:"healthCheck"`
// Cors contains the route cors headers
Cors Cors `yaml:"cors"`
RateLimit int `yaml:"rateLimit"`
@@ -208,8 +209,8 @@ type Gateway struct {
type RouteHealthCheck struct {
Path string `yaml:"path"`
Interval int `yaml:"interval"`
Timeout int `yaml:"timeout"`
Interval string `yaml:"interval"`
Timeout string `yaml:"timeout"`
HealthyStatuses []int `yaml:"healthyStatuses"`
}
type GatewayConfig struct {
@@ -275,3 +276,12 @@ type JWTSecret struct {
ISS string `yaml:"iss"`
Secret string `yaml:"secret"`
}
// Health represents the health check content for a route
type Health struct {
Name string
URL string
TimeOut time.Duration
Interval string
HealthyStatuses []int
}

View File

@@ -11,3 +11,5 @@ const JWTAuth = "jwt" // JWT authentication middleware
const OAuth = "oauth" // OAuth authentication middleware
// Round-robin counter
var counter uint32
var Routes *[]Route

View File

@@ -21,6 +21,7 @@ import (
"log"
"os"
"runtime"
"strings"
"github.com/jkaninda/goma-gateway/util"
)
@@ -59,7 +60,7 @@ func Fatal(msg string, args ...interface{}) {
func Debug(msg string, args ...interface{}) {
log.SetOutput(getStd(util.GetStringEnv("GOMA_ACCESS_LOG", "/dev/stdout")))
logLevel := util.GetStringEnv("GOMA_LOG_LEVEL", "")
if logLevel == "trace" || logLevel == "debug" {
if strings.ToLower(logLevel) == "trace" || strings.ToLower(logLevel) == "debug" {
logWithCaller("DEBUG", msg, args...)
}
@@ -67,7 +68,7 @@ func Debug(msg string, args ...interface{}) {
func Trace(msg string, args ...interface{}) {
log.SetOutput(getStd(util.GetStringEnv("GOMA_ACCESS_LOG", "/dev/stdout")))
logLevel := util.GetStringEnv("GOMA_LOG_LEVEL", "")
if logLevel == "trace" {
if strings.ToLower(logLevel) == "trace" {
logWithCaller("DEBUG", msg, args...)
}
@@ -89,11 +90,13 @@ func logWithCaller(level, msg string, args ...interface{}) {
}
// Log message with caller information if GOMA_LOG_LEVEL is trace
logLevel := util.GetStringEnv("GOMA_LOG_LEVEL", "")
if logLevel == "trace" {
if strings.ToLower(logLevel) != "off" {
if strings.ToLower(logLevel) == "trace" {
log.Printf("%s: %s (File: %s, Line: %d)\n", level, formattedMessage, file, line)
} else {
log.Printf("%s: %s\n", level, formattedMessage)
}
}
}
func getStd(out string) *os.File {

View File

@@ -10,11 +10,13 @@ You may get a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
*/
import (
"github.com/robfig/cron/v3"
"net/url"
"os"
"regexp"
"strconv"
"strings"
"time"
)
// FileExists checks if the file does exist
@@ -122,3 +124,21 @@ func UrlParsePath(uri string) string {
func HasWhitespace(s string) bool {
return regexp.MustCompile(`\s`).MatchString(s)
}
// IsValidCronExpression verify cronExpression and returns boolean
func IsValidCronExpression(cronExpr string) bool {
// Parse the cron expression
_, err := cron.ParseStandard(cronExpr)
return err == nil
}
func ParseDuration(durationStr string) (time.Duration, error) {
if durationStr == "" {
return 0, nil
}
duration, err := time.ParseDuration(durationStr)
if err != nil {
return 0, err
}
return duration, nil
}

View File

@@ -1 +1,35 @@
package util
import (
"log"
"testing"
"time"
)
func TestConExpression(t *testing.T) {
cronExpression := "@every 30s"
if !IsValidCronExpression(cronExpression) {
t.Fatal("Cron expression should be valid")
}
log.Println(" Cron is valid")
}
func TestParseDuration(t *testing.T) {
d1, err1 := ParseDuration("20s")
if err1 != nil {
t.Error("Error:", err1)
} else {
log.Printf("Parsed duration: %d", d1)
log.Printf("Time out: %s\n", time.Now().Add(d1))
}
d2, err2 := ParseDuration("10m")
if err2 != nil {
t.Errorf("Error: %v", err2)
} else {
log.Printf("Parsed duration: %d\n", d2)
log.Printf("Time out: %s\n", time.Now().Add(d2))
}
}