From 809e08469801963123a776326d265bed300b2789 Mon Sep 17 00:00:00 2001 From: martabal <74269598+martabal@users.noreply.github.com> Date: Thu, 19 Oct 2023 20:19:59 +0200 Subject: [PATCH] refactor: simplify data registration --- .github/dependabot.yml | 6 ++ .github/workflows/docker.yml | 26 +++++--- .github/workflows/test.yml | 10 +-- Dockerfile | 2 +- go.mod | 15 ++--- go.sum | 31 ++++----- package.json | 9 +-- src/immich/data.go | 66 ++++++++++--------- src/init.go | 2 +- .../prometheus.go} | 56 +++++++++------- src/tests/prometheus_test.go | 35 ++++++++++ 11 files changed, 158 insertions(+), 100 deletions(-) create mode 100644 .github/dependabot.yml rename src/{immich/sendbackmessage.go => prometheus/prometheus.go} (77%) create mode 100644 src/tests/prometheus_test.go diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..9096371 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2c506cf..ad48f1a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,7 +3,7 @@ on: workflow_dispatch: inputs: tags: - description: 'version' + description: "version" required: true type: string @@ -14,9 +14,20 @@ jobs: - name: Checkout uses: actions/checkout@v3 with: - ref: 'main' + ref: "main" fetch-depth: 0 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + tags: | + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + type=sha + type=raw,value=${{ inputs.tags }} + - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -27,9 +38,9 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v2 with: - username: martabal + username: ${{ github.repository_owner }} password: ${{ secrets.DOCKERHUB_TOKEN }} - + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -43,8 +54,5 @@ jobs: context: ./ platforms: linux/arm/v7,linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} - tags: | - martabal/immich-exporter:${{ inputs.tags }} - martabal/immich-exporter:latest - ghcr.io/${{ github.repository_owner }}/immich-exporter:${{ inputs.tags }} - ghcr.io/${{ github.repository_owner }}/immich-exporter:latest \ No newline at end of file + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f73fbd..0c5b55e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,16 +2,18 @@ name: Test on: workflow_dispatch: push: - branches: [ "main" ] + branches: ["main"] pull_request: - branches: [ "main" ] -jobs: + branches: ["main"] +jobs: test: name: Run tests runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 + - name: Run unit tests + run: go test -v ./src/tests - name: Run formatter run: test -z $(gofmt -l ./src) \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d611b63..d76989f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20-alpine3.18 AS builder +FROM golang:1.21-alpine3.18 AS builder WORKDIR /app diff --git a/go.mod b/go.mod index 1e30bae..c5f7ce6 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,20 @@ module immich-exp -go 1.20 +go 1.21 require ( github.com/joho/godotenv v1.5.1 - github.com/prometheus/client_golang v1.16.0 + github.com/prometheus/client_golang v1.17.0 github.com/sirupsen/logrus v1.9.3 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.0 // indirect - golang.org/x/sys v0.10.0 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + golang.org/x/sys v0.13.0 // indirect google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/go.sum b/go.sum index 09c76d3..6934ef0 100644 --- a/go.sum +++ b/go.sum @@ -5,40 +5,35 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk= -github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/package.json b/package.json index c9585c9..1f69380 100644 --- a/package.json +++ b/package.json @@ -1,14 +1,15 @@ { "name": "immich-exporter", - "version": "1.0.0", + "version": "1.0.1", "description": "exporter for immich", "main": "src/main.go", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1", + "build" : "go build -o ./immich-exporter.out ./src && ./immich-exporter.out -e", + "build:env" : "go build -o ./immich-exporter.out ./src && ./immich-exporter.out -e", "dev" : "go run ./src", "dev:env" : "go run ./src -e", - "build" : "go build -o ./immich-exporter.out ./src && ./immich-exporter.out -e", - "build:env" : "go build -o ./immich-exporter.out ./src && ./immich-exporter.out -e" + "test": "go test -v ./src/tests", + "update": "go get -u ./src && go mod tidy" }, "keywords": [ "exporter", diff --git a/src/immich/data.go b/src/immich/data.go index 35420e4..0230296 100644 --- a/src/immich/data.go +++ b/src/immich/data.go @@ -5,15 +5,21 @@ import ( "fmt" "immich-exp/src/models" "io/ioutil" - "log" + "net/http" "sync" + prom "immich-exp/src/prometheus" + + log "github.com/sirupsen/logrus" + "github.com/prometheus/client_golang/prometheus" ) var wg sync.WaitGroup +var unmarshalError = "Can not unmarshal JSON" + func Allrequests(r *prometheus.Registry) { wg.Add(1) @@ -39,7 +45,7 @@ func Analyze(r *prometheus.Registry) { if err != nil && err2 != nil { } else { - SendBackMessagePreference(res2, res1, r) + prom.SendBackMessagePreference(res2, res1, r) } close(serverinfo) close(allusers) @@ -59,7 +65,7 @@ func GetAllUsers(c chan func() (*models.StructAllUsers, error)) { result := new(models.StructAllUsers) if err := json.Unmarshal(body, &result); err != nil { // Parse []byte to go struct pointer - log.Println("Can not unmarshal JSON") + log.Error(unmarshalError) } c <- (func() (*models.StructAllUsers, error) { return result, nil }) @@ -81,10 +87,10 @@ func ServerVersion(r *prometheus.Registry) { var result models.StructServerVersion if err := json.Unmarshal(body, &result); err != nil { // Parse []byte to go struct pointer - log.Println("Can not unmarshal JSON for version") + log.Error(unmarshalError) } - SendBackMessageserverVersion(&result, r) + prom.SendBackMessageserverVersion(&result, r) } } } @@ -92,16 +98,8 @@ func ServerVersion(r *prometheus.Registry) { func ServerInfo(c chan func() (*models.StructServerInfo, error)) { defer wg.Done() resp, err := Apirequest("/api/server-info/stats", "GET") - if err != nil { - if err.Error() == "403" { - log.Println("Cookie changed, try to reconnect ...") - } else { - if models.GetPromptError() == false { - log.Println("Error : ", err) - } - } + if err == nil { - } else { if models.GetPromptError() == true { models.SetPromptError(false) } @@ -112,7 +110,7 @@ func ServerInfo(c chan func() (*models.StructServerInfo, error)) { result := new(models.StructServerInfo) if err := json.Unmarshal(body, &result); err != nil { // Parse []byte to go struct pointer - log.Println("Can not unmarshal JSON for server infos") + log.Println(unmarshalError) } c <- (func() (*models.StructServerInfo, error) { return result, nil }) @@ -124,33 +122,41 @@ func Apirequest(uri string, method string) (*http.Response, error) { req, err := http.NewRequest(method, models.Getbaseurl()+uri, nil) if err != nil { - log.Fatalln("Error with url") + log.Fatal("Error with url") } req.Header.Add("Accept", "application/json") req.Header.Add("x-api-key", models.GetApiKey()) client := &http.Client{} resp, err := client.Do(req) if err != nil { - fmt.Println(err) err := fmt.Errorf("Can't connect to server") if models.GetPromptError() == false { - log.Println(err.Error()) + log.Error(err.Error()) models.SetPromptError(true) } return resp, err - } else { - if resp.StatusCode == 200 { - models.SetPromptError(false) - return resp, nil - } else { - err := fmt.Errorf("%d", resp.StatusCode) - if models.GetPromptError() == false { - models.SetPromptError(true) - log.Println("Error code", err.Error(), " for ", models.Getbaseurl()+uri) - } - return resp, err - } } + switch resp.StatusCode { + case http.StatusOK: + if models.GetPromptError() { + models.SetPromptError(false) + } + return resp, nil + case http.StatusUnauthorized, http.StatusForbidden: + err := fmt.Errorf("%d", resp.StatusCode) + + log.Fatal("Api key unauthorized") + + return resp, err + default: + err := fmt.Errorf("%d", resp.StatusCode) + if !models.GetPromptError() { + models.SetPromptError(true) + log.Debug("Error code ", resp.StatusCode) + } + return resp, err + } + } diff --git a/src/init.go b/src/init.go index bca5e4f..2ef4a20 100644 --- a/src/init.go +++ b/src/init.go @@ -4,7 +4,7 @@ import ( "encoding/json" "flag" "fmt" - "immich-exp/src/immich" + immich "immich-exp/src/immich" "immich-exp/src/models" "net/http" diff --git a/src/immich/sendbackmessage.go b/src/prometheus/prometheus.go similarity index 77% rename from src/immich/sendbackmessage.go rename to src/prometheus/prometheus.go index 65c869e..3653e1c 100644 --- a/src/immich/sendbackmessage.go +++ b/src/prometheus/prometheus.go @@ -1,29 +1,30 @@ -package immich +package prom import ( "immich-exp/src/models" "strconv" + "strings" "github.com/prometheus/client_golang/prometheus" ) +type Gauge []struct { + name string + help string + value float64 +} + func SendBackMessagePreference(result *models.StructServerInfo, result2 *models.StructAllUsers, r *prometheus.Registry) { - total_photos := prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "immich_app_total_photos", - Help: "The total number of photos", - }) - total_videos := prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "immich_app_total_videos", - Help: "The total number of videos", - }) - total_usage := prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "immich_app_total_usage", - Help: "The total usage of disk", - }) - total_users := prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "immich_app_number_users", - Help: "The total number of users", - }) + + gauges := Gauge{ + {"total photos", "The total number of photos", float64((*result).Photos)}, + {"total videos", "The total number of videos", float64((*result).Videos)}, + {"total usage", "The max number of active torrents allowed", float64((*result).Usage)}, + {"number users", "The total number of users", float64(len((*result).UsageByUser))}, + } + + register(gauges, r) + user_info := prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "immich_user_info", Help: "All infos about users", @@ -43,17 +44,9 @@ func SendBackMessagePreference(result *models.StructServerInfo, result2 *models. }, []string{"uid", "firstname", "lastname"}) r.MustRegister(user_info) - r.MustRegister(total_usage) - r.MustRegister(total_videos) - r.MustRegister(total_photos) - r.MustRegister(total_users) r.MustRegister(user_usage) r.MustRegister(user_videos) r.MustRegister(user_photos) - total_photos.Add(float64((*result).Photos)) - total_videos.Add(float64((*result).Videos)) - total_usage.Add(float64((*result).Usage)) - total_users.Add(float64(len((*result).UsageByUser))) for i := 0; i < len((*result).UsageByUser); i++ { var myuser = GetName((*result).UsageByUser[i].UserID, result2) @@ -94,3 +87,16 @@ func GetName(result string, result2 *models.StructAllUsers) models.StructCustomU } return myuser } + +func register(gauges Gauge, r *prometheus.Registry) { + for _, gauge := range gauges { + name := "immich_app_" + strings.Replace(gauge.name, " ", "_", -1) + help := gauge.help + g := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: name, + Help: help, + }) + r.MustRegister(g) + g.Set(gauge.value) + } +} diff --git a/src/tests/prometheus_test.go b/src/tests/prometheus_test.go new file mode 100644 index 0000000..929c412 --- /dev/null +++ b/src/tests/prometheus_test.go @@ -0,0 +1,35 @@ +package prom + +import ( + "immich-exp/src/models" + prom "immich-exp/src/prometheus" + "reflect" + "testing" +) + +func TestGetName(t *testing.T) { + result2 := &models.StructAllUsers{ + {ID: "1", FirstName: "John", LastName: "Doe", Email: "john@example.com", IsAdmin: true}, + {ID: "2", FirstName: "Jane", LastName: "Smith", Email: "jane@example.com", IsAdmin: false}, + } + + result := "1" + expected := models.StructCustomUser{ + ID: "1", + FirstName: "John", + LastName: "Doe", + Email: "john@example.com", + IsAdmin: true, + } + actual := prom.GetName(result, result2) + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Expected: %v, but got: %v", expected, actual) + } + + result = "3" + expected = models.StructCustomUser{} + actual = prom.GetName(result, result2) + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Expected: %v, but got: %v", expected, actual) + } +}