From bda504dbe4699979a84828adab6cd5724a573412 Mon Sep 17 00:00:00 2001 From: Galorhallen Date: Sun, 26 Dec 2021 23:36:30 +0100 Subject: [PATCH] Add async metrics fetch for multiple piholes parent 8d5586558c4b3695a64eff488ff13fb747f7b087 author Galorhallen 1640558190 +0100 committer Galorhallen 1640821760 +0100 Add test for multiple pihole Add async mode for multiple piholes Fixed GitHub Actions go versions Add test for multiple pihole Cleanup --- go.mod | 1 + go.sum | 7 +++++ internal/pihole/client.go | 61 +++++++++++++++++++++++++-------------- internal/server/server.go | 55 +++++++++++++++++++++++------------ 4 files changed, 84 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index 990273e..fc899ad 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/xonvanetta/shutdown v0.0.3 golang.org/x/net v0.0.0-20200625001655-4c5254603344 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c ) require ( diff --git a/go.sum b/go.sum index 4ef601a..155cdf4 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -195,6 +197,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -242,6 +245,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -293,6 +298,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= diff --git a/internal/pihole/client.go b/internal/pihole/client.go index 789dd60..f453a62 100644 --- a/internal/pihole/client.go +++ b/internal/pihole/client.go @@ -16,12 +16,38 @@ import ( "github.com/eko/pihole-exporter/internal/metrics" ) +type ClientStatus byte + +const ( + MetricsCollectionInProgress ClientStatus = iota + MetricsCollectionSuccess + MetricsCollectionError + MetricsCollectionTimeout +) + +func (status ClientStatus) String() string { + return []string{"MetricsCollectionInProgress", "MetricsCollectionSuccess", "MetricsCollectionError", "MetricsCollectionTimeout"}[status] +} + +type ClientChannel struct { + Status ClientStatus + Err error +} + +func (c *ClientChannel) String() string { + if c.Err != nil { + return fmt.Sprintf("ClientChannel", c.Status, c.Err.Error()) + } else { + return fmt.Sprintf("ClientChannel>", c.Status) + } +} + // Client struct is a PI-Hole client to request an instance of a PI-Hole ad blocker. type Client struct { - httpClient http.Client - interval time.Duration - config *config.Config - MetricRetrieved chan bool + httpClient http.Client + interval time.Duration + config *config.Config + Status chan *ClientChannel } // NewClient method initializes a new PI-Hole client. @@ -41,6 +67,7 @@ func NewClient(config *config.Config) *Client { return http.ErrUseLastResponse }, }, + Status: make(chan *ClientChannel, 1), } } @@ -48,32 +75,24 @@ func (c *Client) String() string { return c.config.PIHoleHostname } -/* -// Metrics scrapes pihole and sets them -func (c *Client) Metrics() http.HandlerFunc { - return func(writer http.ResponseWriter, request *http.Request) { - stats, err := c.getStatistics() - if err != nil { - writer.WriteHeader(http.StatusBadRequest) - _, _ = writer.Write([]byte(err.Error())) - return - } +func (c *Client) CollectMetricsAsync(writer http.ResponseWriter, request *http.Request) { + log.Printf("Collecting from %s", c.config.PIHoleHostname) + if stats, err := c.getStatistics(); err == nil { c.setMetrics(stats) - - log.Printf("New tick of statistics: %s", stats.ToString()) - promhttp.Handler().ServeHTTP(writer, request) + c.Status <- &ClientChannel{Status: MetricsCollectionSuccess, Err: nil} + log.Printf("New tick of statistics from %s: %s", c.config.PIHoleHostname, stats) + } else { + c.Status <- &ClientChannel{Status: MetricsCollectionError, Err: err} } -}*/ +} func (c *Client) CollectMetrics(writer http.ResponseWriter, request *http.Request) error { - stats, err := c.getStatistics() if err != nil { return err } c.setMetrics(stats) - - log.Printf("New tick of statistics from %s: %s", c, stats) + log.Printf("New tick of statistics from %s: %s", c.config.PIHoleHostname, stats) return nil } diff --git a/internal/server/server.go b/internal/server/server.go index 55968b5..3c5efd4 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -22,32 +22,28 @@ type Server struct { // the different routes that will be used by Prometheus (metrics) or for monitoring (readiness, liveness). func NewServer(port uint16, clients []*pihole.Client) *Server { mux := http.NewServeMux() - httpServer := &http.Server{Addr: ":" + strconv.Itoa(int(port)), Handler: mux} + httpServer := &http.Server{ + Addr: ":" + strconv.Itoa(int(port)), + Handler: mux, + } s := &Server{ httpServer: httpServer, } - mux.HandleFunc("/metrics", - func(writer http.ResponseWriter, request *http.Request) { - errors := make([]string, 0) + mux.HandleFunc("/metrics", func(writer http.ResponseWriter, request *http.Request) { + log.Printf("request.Header: %v\n", request.Header) + + for _, client := range clients { + go client.CollectMetricsAsync(writer, request) + } - for _, client := range clients { - if err := client.CollectMetrics(writer, request); err != nil { - errors = append(errors, err.Error()) - fmt.Printf("Error %s\n", err) - } - } + for _, client := range clients { + log.Printf("Received %s from %s\n", <-client.Status, client.GetHostname()) + } - if len(errors) == len(clients) { - writer.WriteHeader(http.StatusBadRequest) - body := strings.Join(errors, "\n") - _, _ = writer.Write([]byte(body)) - } - - promhttp.Handler().ServeHTTP(writer, request) - }, - ) + promhttp.Handler().ServeHTTP(writer, request) + }) mux.Handle("/readiness", s.readinessHandler()) mux.Handle("/liveness", s.livenessHandler()) @@ -73,6 +69,27 @@ func (s *Server) Stop() { s.httpServer.Shutdown(ctx) } +func (s *Server) handleMetrics(clients []*pihole.Client) http.HandlerFunc { + return func(writer http.ResponseWriter, request *http.Request) { + errors := make([]string, 0) + + for _, client := range clients { + if err := client.CollectMetrics(writer, request); err != nil { + errors = append(errors, err.Error()) + fmt.Printf("Error %s\n", err) + } + } + + if len(errors) == len(clients) { + writer.WriteHeader(http.StatusBadRequest) + body := strings.Join(errors, "\n") + _, _ = writer.Write([]byte(body)) + } + + promhttp.Handler().ServeHTTP(writer, request) + } +} + func (s *Server) readinessHandler() http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { if s.isReady() {