From f2cc1888334a1674ca163606b0385041aac220a5 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Mon, 3 Jul 2023 00:16:08 +0200 Subject: [PATCH] Initial commit Signed-off-by: Michael Hanselmann --- .github/dependabot.yml | 14 +++ .github/workflows/ci.yaml | 19 ++++ .github/workflows/release.yaml | 17 ++++ .gitignore | 2 + .goreleaser.yml | 73 ++++++++++++++ LICENSE | 27 ++++++ README.md | 57 +++++++++++ collector.go | 22 +++++ collector_test.go | 31 ++++++ contrib/Dockerfile.goreleaser | 15 +++ contrib/build-all | 16 +++ correspondent.go | 66 +++++++++++++ correspondent_test.go | 107 +++++++++++++++++++++ documenttype.go | 58 +++++++++++ documenttype_test.go | 92 ++++++++++++++++++ go.mod | 42 ++++++++ go.sum | 101 +++++++++++++++++++ internal/ref/ref.go | 8 ++ internal/testutil/metrics.go | 34 +++++++ log.go | 147 ++++++++++++++++++++++++++++ log_test.go | 171 +++++++++++++++++++++++++++++++++ main.go | 65 +++++++++++++ multicollector.go | 60 ++++++++++++ storagepath.go | 58 +++++++++++ storagepath_test.go | 92 ++++++++++++++++++ tag.go | 58 +++++++++++ tag_test.go | 98 +++++++++++++++++++ task.go | 126 ++++++++++++++++++++++++ task_test.go | 100 +++++++++++++++++++ 29 files changed, 1776 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .gitignore create mode 100644 .goreleaser.yml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 collector.go create mode 100644 collector_test.go create mode 100644 contrib/Dockerfile.goreleaser create mode 100755 contrib/build-all create mode 100644 correspondent.go create mode 100644 correspondent_test.go create mode 100644 documenttype.go create mode 100644 documenttype_test.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/ref/ref.go create mode 100644 internal/testutil/metrics.go create mode 100644 log.go create mode 100644 log_test.go create mode 100644 main.go create mode 100644 multicollector.go create mode 100644 storagepath.go create mode 100644 storagepath_test.go create mode 100644 tag.go create mode 100644 tag_test.go create mode 100644 task.go create mode 100644 task_test.go diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..a4149b8 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +--- +version: 2 +updates: + - package-ecosystem: gomod + directory: / + schedule: + interval: weekly + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + +# vim: set sw=2 sts=2 et : diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..32726da --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,19 @@ +name: Run tests + +on: + workflow_dispatch: + pull_request: + push: + schedule: + - cron: '13 4 */9 * *' + +permissions: + contents: read + +jobs: + test: + uses: hansmi/ghactions-go-test-workflow/.github/workflows/test.yaml@stable + with: + runs-on: ubuntu-latest + +# vim: set sw=2 sts=2 et : diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..d367dba --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,17 @@ +name: Release + +on: + workflow_dispatch: + pull_request: + push: + +permissions: {} + +jobs: + release: + uses: hansmi/ghactions-goreleaser-workflow/.github/workflows/release.yaml@stable + permissions: + contents: write + packages: write + +# vim: set sw=2 sts=2 et : diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..99ca961 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/dist/ +/prometheus-paperless-exporter diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..cca57b3 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,73 @@ +# Configuration for GoReleaser +# https://goreleaser.com/ +# +# Local test: contrib/build-all +# + +project_name: prometheus-paperless-exporter + +builds: + - main: . + binary: prometheus-paperless-exporter + env: + - CGO_ENABLED=0 + targets: + - go_first_class + flags: + - -trimpath + ldflags: | + -s -w + -X github.com/prometheus/common/version.Version={{.Version}} + -X github.com/prometheus/common/version.Revision={{.FullCommit}} + -X github.com/prometheus/common/version.Branch={{.Branch}} + -X github.com/prometheus/common/version.BuildDate={{.Date}} + +nfpms: + - description: Prometheus metrics for Paperless-ngx + maintainer: M. Hanselmann + bindir: /usr/bin + license: BSD-3-Clause + formats: + - deb + - rpm + contents: + - src: ./README.md + dst: /usr/share/doc/prometheus-paperless-exporter/README.md + - src: ./LICENSE + dst: /usr/share/doc/prometheus-paperless-exporter/LICENSE + +archives: + - format: tar.gz + wrap_in_directory: true + files: + - LICENSE + - README.md + +dockers: + - ids: + - prometheus-paperless-exporter + use: buildx + dockerfile: contrib/Dockerfile.goreleaser + extra_files: + - LICENSE + - README.md + image_templates: + - ghcr.io/hansmi/prometheus-paperless-exporter:{{.Tag}} + - ghcr.io/hansmi/prometheus-paperless-exporter:v{{.Major}} + - ghcr.io/hansmi/prometheus-paperless-exporter:latest + build_flag_templates: + - --pull + - --label=org.opencontainers.image.created={{.Date}} + - --label=org.opencontainers.image.name={{.ProjectName}} + - --label=org.opencontainers.image.revision={{.FullCommit}} + - --label=org.opencontainers.image.version={{.Version}} + - --label=org.opencontainers.image.source={{.GitURL}} + +release: + draft: true + prerelease: auto + +snapshot: + name_template: '{{ incpatch .Version }}-snapshot{{ replace (replace .Date ":" "") "-" "" }}+g{{ .ShortCommit }}' + +# vim: set sw=2 sts=2 et : diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9d7d9b1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2023 Michael Hanselmann. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f8fb899 --- /dev/null +++ b/README.md @@ -0,0 +1,57 @@ +# Paperless-ngx metrics for Prometheus + +[![Latest release](https://img.shields.io/github/v/release/hansmi/prometheus-paperless-exporter)][releases] +[![Release workflow](https://github.com/hansmi/prometheus-paperless-exporter/actions/workflows/release.yaml/badge.svg)](https://github.com/hansmi/prometheus-paperless-exporter/actions/workflows/release.yaml) +[![CI workflow](https://github.com/hansmi/prometheus-paperless-exporter/actions/workflows/ci.yaml/badge.svg)](https://github.com/hansmi/prometheus-paperless-exporter/actions/workflows/ci.yaml) +[![Go reference](https://pkg.go.dev/badge/github.com/hansmi/prometheus-paperless-exporter.svg)](https://pkg.go.dev/github.com/hansmi/prometheus-paperless-exporter) + +This repository hosts a Prometheus metrics exporter for +[Paperless-ngx][paperless], a document management system transforming physical +documents into a searchable online archive. The exporter relies on [Paperless' +REST API][paperless-api]. + +An implementation using the API was chosen to provide the same perspective as +web browsers. + + +## Usage + +`prometheus-paperless-exporter` listens on TCP port 8081 by default. To listen on +another address use the `-web.listen-address` flag (e.g. +`-web.listen-address=127.0.0.1:3000`). + +TLS and HTTP basic authentication is supported through the [Prometheus exporter +toolkit][toolkit]. A configuration file can be passed to the `-web.config` flag +([documentation][toolkitconfig]). + +See the `--help` output for more flags. + + +## Installation + +Pre-built binaries are provided for [all releases][releases]: + +* Binary archives (`.tar.gz`) +* Debian/Ubuntu (`.deb`) +* RHEL/Fedora (`.rpm`) +* Microsoft Windows (`.zip`) + +Docker images via GitHub's container registry: + +```shell +docker pull ghcr.io/hansmi/prometheus-paperless-exporter +``` + +With the source being available it's also possible to produce custom builds +directly using [Go][golang] or [GoReleaser][goreleaser]. + + +[golang]: https://golang.org/ +[goreleaser]: https://goreleaser.com/ +[paperless-api]: https://docs.paperless-ngx.com/api/ +[paperless]: https://docs.paperless-ngx.com/ +[releases]: https://github.com/hansmi/prometheus-paperless-exporter/releases/latest +[toolkit]: https://github.com/prometheus/exporter-toolkit +[toolkitconfig]: https://github.com/prometheus/exporter-toolkit/blob/master/docs/web-configuration.md + + diff --git a/collector.go b/collector.go new file mode 100644 index 0000000..6b5bb5b --- /dev/null +++ b/collector.go @@ -0,0 +1,22 @@ +package main + +import ( + "time" + + "github.com/hansmi/paperhooks/pkg/client" + "github.com/prometheus/client_golang/prometheus" +) + +func newCollector(cl *client.Client, timeout time.Duration) prometheus.Collector { + return &multiCollector{ + timeout: timeout, + members: []multiCollectorMember{ + newTagCollector(cl), + newCorrespondentCollector(cl), + newDocumentTypeCollector(cl), + newStoragePathCollector(cl), + newTaskCollector(cl), + newLogCollector(cl), + }, + } +} diff --git a/collector_test.go b/collector_test.go new file mode 100644 index 0000000..65a830f --- /dev/null +++ b/collector_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/hansmi/paperhooks/pkg/client" + "github.com/hansmi/prometheus-paperless-exporter/internal/testutil" +) + +func TestCollector(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "{}") + })) + t.Cleanup(ts.Close) + + cl := client.New(client.Options{ + BaseURL: ts.URL, + }) + + c := newCollector(cl, time.Minute) + + testutil.CollectAndCompare(t, c, ` +# HELP paperless_task_status_info Task status names. +# TYPE paperless_task_status_info gauge +paperless_task_status_info{status="success"} 1 +`) +} diff --git a/contrib/Dockerfile.goreleaser b/contrib/Dockerfile.goreleaser new file mode 100644 index 0000000..e78fe17 --- /dev/null +++ b/contrib/Dockerfile.goreleaser @@ -0,0 +1,15 @@ +FROM docker.io/library/alpine:latest + +RUN apk add --no-cache tzdata + +LABEL org.opencontainers.image.licenses=BSD-3-Clause +LABEL org.opencontainers.image.description="Prometheus metrics for Paperless-ngx" + +WORKDIR / + +COPY LICENSE README.md / +COPY prometheus-paperless-exporter / + +ENTRYPOINT ["/prometheus-paperless-exporter"] + +# vim: set ft=dockerfile : diff --git a/contrib/build-all b/contrib/build-all new file mode 100755 index 0000000..0d56878 --- /dev/null +++ b/contrib/build-all @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e -u -o pipefail + +package=github.com/hansmi/prometheus-paperless-exporter +docker_gid=$(getent group docker | cut -d: -f3) + +docker run --rm \ + --user "$(id -u):$(id -g)" --group-add="$docker_gid" \ + --env HOME=/tmp \ + -v "${PWD}:/go/src/${package}" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -w "/go/src/${package}" \ + goreleaser/goreleaser:latest release --snapshot --clean --skip-publish + +# vim: set sw=2 sts=2 et : diff --git a/correspondent.go b/correspondent.go new file mode 100644 index 0000000..6987f9d --- /dev/null +++ b/correspondent.go @@ -0,0 +1,66 @@ +package main + +import ( + "context" + "strconv" + + "github.com/hansmi/paperhooks/pkg/client" + "github.com/prometheus/client_golang/prometheus" +) + +type correspondentClient interface { + ListAllCorrespondents(context.Context, *client.ListCorrespondentsOptions, func(context.Context, client.Correspondent) error) error +} + +type correspondentCollector struct { + cl correspondentClient + + infoDesc *prometheus.Desc + docCountDesc *prometheus.Desc + lastCorrespondenceDesc *prometheus.Desc +} + +func newCorrespondentCollector(cl correspondentClient) *correspondentCollector { + return &correspondentCollector{ + cl: cl, + + infoDesc: prometheus.NewDesc("paperless_correspondent_info", + "Static information about a correspondent.", + []string{"id", "name", "slug"}, nil), + docCountDesc: prometheus.NewDesc("paperless_correspondent_document_count", + "Number of documents associated with a correspondent.", + []string{"id"}, nil), + lastCorrespondenceDesc: prometheus.NewDesc("paperless_correspondent_last_correspondence_timestamp_seconds", + "Number of seconds since 1970 of the most recent correspondence.", + []string{"id"}, nil), + } +} + +func (c *correspondentCollector) describe(ch chan<- *prometheus.Desc) { + ch <- c.infoDesc + ch <- c.docCountDesc + ch <- c.lastCorrespondenceDesc +} + +func (c *correspondentCollector) collect(ctx context.Context, ch chan<- prometheus.Metric) error { + opts := &client.ListCorrespondentsOptions{} + opts.Ordering.Field = "name" + + return c.cl.ListAllCorrespondents(ctx, opts, func(_ context.Context, correspondent client.Correspondent) error { + id := strconv.FormatInt(correspondent.ID, 10) + + ch <- prometheus.MustNewConstMetric(c.infoDesc, prometheus.GaugeValue, 1, + id, + correspondent.Name, + correspondent.Slug, + ) + + ch <- prometheus.MustNewConstMetric(c.docCountDesc, prometheus.GaugeValue, + float64(correspondent.DocumentCount), id) + + ch <- prometheus.MustNewConstMetric(c.lastCorrespondenceDesc, prometheus.GaugeValue, + optionalTimestamp(correspondent.LastCorrespondence), id) + + return nil + }) +} diff --git a/correspondent_test.go b/correspondent_test.go new file mode 100644 index 0000000..6bc7c37 --- /dev/null +++ b/correspondent_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/hansmi/paperhooks/pkg/client" + "github.com/hansmi/prometheus-paperless-exporter/internal/ref" + "github.com/hansmi/prometheus-paperless-exporter/internal/testutil" +) + +type fakeCorrespondentClient struct { + items []client.Correspondent + err error +} + +func (c *fakeCorrespondentClient) ListAllCorrespondents(ctx context.Context, opts *client.ListCorrespondentsOptions, handler func(context.Context, client.Correspondent) error) error { + for _, i := range c.items { + if err := handler(ctx, i); err != nil { + return err + } + } + + return c.err +} + +func TestCorrespondent(t *testing.T) { + errTest := errors.New("test error") + + for _, tc := range []struct { + name string + cl fakeCorrespondentClient + wantErr error + }{ + { + name: "empty", + }, + { + name: "listing fails", + cl: fakeCorrespondentClient{ + err: errTest, + }, + wantErr: errTest, + }, + { + name: "correspondents", + cl: fakeCorrespondentClient{ + items: []client.Correspondent{ + {ID: 21383}, + {ID: 3096}, + {ID: 22044}, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + c := newCorrespondentCollector(&tc.cl) + + err := c.collect(context.Background(), testutil.DiscardMetrics(t)) + + if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Error diff (-want +got):\n%s", diff) + } + }) + } +} + +func TestCorrespondentCollect(t *testing.T) { + cl := fakeCorrespondentClient{} + + c := newMultiCollector(newCorrespondentCollector(&cl)) + + testutil.CollectAndCompare(t, c, ``) + + cl.items = append(cl.items, []client.Correspondent{ + {ID: 15818, Name: "bank", Slug: "aslug"}, + { + ID: 167, + Name: "insurance", + DocumentCount: 2, + LastCorrespondence: ref.Ref(time.Date(2019, time.July, 1, 0, 0, 0, 0, time.UTC)), + }, + {ID: 24467, Name: "employer", DocumentCount: 121}, + }...) + + testutil.CollectAndCompare(t, c, ` +# HELP paperless_correspondent_document_count Number of documents associated with a correspondent. +# TYPE paperless_correspondent_document_count gauge +paperless_correspondent_document_count{id="15818"} 0 +paperless_correspondent_document_count{id="167"} 2 +paperless_correspondent_document_count{id="24467"} 121 +# HELP paperless_correspondent_info Static information about a correspondent. +# TYPE paperless_correspondent_info gauge +paperless_correspondent_info{id="15818",name="bank",slug="aslug"} 1 +paperless_correspondent_info{id="167",name="insurance",slug=""} 1 +paperless_correspondent_info{id="24467",name="employer",slug=""} 1 +# HELP paperless_correspondent_last_correspondence_timestamp_seconds Number of seconds since 1970 of the most recent correspondence. +# TYPE paperless_correspondent_last_correspondence_timestamp_seconds gauge +paperless_correspondent_last_correspondence_timestamp_seconds{id="15818"} 0 +paperless_correspondent_last_correspondence_timestamp_seconds{id="167"} 1.5619392e+09 +paperless_correspondent_last_correspondence_timestamp_seconds{id="24467"} 0 +`) +} diff --git a/documenttype.go b/documenttype.go new file mode 100644 index 0000000..65fab05 --- /dev/null +++ b/documenttype.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "strconv" + + "github.com/hansmi/paperhooks/pkg/client" + "github.com/prometheus/client_golang/prometheus" +) + +type documentTypeClient interface { + ListAllDocumentTypes(context.Context, *client.ListDocumentTypesOptions, func(context.Context, client.DocumentType) error) error +} + +type documentTypeCollector struct { + cl documentTypeClient + + infoDesc *prometheus.Desc + docCountDesc *prometheus.Desc +} + +func newDocumentTypeCollector(cl documentTypeClient) *documentTypeCollector { + return &documentTypeCollector{ + cl: cl, + + infoDesc: prometheus.NewDesc("paperless_document_type_info", + "Static information about a document type.", + []string{"id", "name", "slug"}, nil), + docCountDesc: prometheus.NewDesc("paperless_document_type_document_count", + "Number of documents associated with a document type.", + []string{"id"}, nil), + } +} + +func (c *documentTypeCollector) describe(ch chan<- *prometheus.Desc) { + ch <- c.infoDesc + ch <- c.docCountDesc +} + +func (c *documentTypeCollector) collect(ctx context.Context, ch chan<- prometheus.Metric) error { + opts := &client.ListDocumentTypesOptions{} + opts.Ordering.Field = "name" + + return c.cl.ListAllDocumentTypes(ctx, opts, func(_ context.Context, doctype client.DocumentType) error { + id := strconv.FormatInt(doctype.ID, 10) + + ch <- prometheus.MustNewConstMetric(c.infoDesc, prometheus.GaugeValue, 1, + id, + doctype.Name, + doctype.Slug, + ) + + ch <- prometheus.MustNewConstMetric(c.docCountDesc, prometheus.GaugeValue, + float64(doctype.DocumentCount), id) + + return nil + }) +} diff --git a/documenttype_test.go b/documenttype_test.go new file mode 100644 index 0000000..b5c841b --- /dev/null +++ b/documenttype_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/hansmi/paperhooks/pkg/client" + "github.com/hansmi/prometheus-paperless-exporter/internal/testutil" +) + +type fakeDocumentTypeClient struct { + items []client.DocumentType + err error +} + +func (c *fakeDocumentTypeClient) ListAllDocumentTypes(ctx context.Context, opts *client.ListDocumentTypesOptions, handler func(context.Context, client.DocumentType) error) error { + for _, i := range c.items { + if err := handler(ctx, i); err != nil { + return err + } + } + + return c.err +} + +func TestDocumentType(t *testing.T) { + errTest := errors.New("test error") + + for _, tc := range []struct { + name string + cl fakeDocumentTypeClient + wantErr error + }{ + { + name: "empty", + }, + { + name: "listing fails", + cl: fakeDocumentTypeClient{ + err: errTest, + }, + wantErr: errTest, + }, + { + name: "documentTypes", + cl: fakeDocumentTypeClient{ + items: []client.DocumentType{ + {ID: 23758}, + {ID: 22848}, + {ID: 10504}, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + c := newDocumentTypeCollector(&tc.cl) + + err := c.collect(context.Background(), testutil.DiscardMetrics(t)) + + if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Error diff (-want +got):\n%s", diff) + } + }) + } +} + +func TestDocumentTypeCollect(t *testing.T) { + cl := fakeDocumentTypeClient{} + + c := newMultiCollector(newDocumentTypeCollector(&cl)) + + testutil.CollectAndCompare(t, c, ``) + + cl.items = append(cl.items, []client.DocumentType{ + {ID: 3760, Name: "Contract", Slug: "contract"}, + {ID: 5558, Name: "Purchase order", Slug: "po", DocumentCount: 20}, + }...) + + testutil.CollectAndCompare(t, c, ` +# HELP paperless_document_type_document_count Number of documents associated with a document type. +# TYPE paperless_document_type_document_count gauge +paperless_document_type_document_count{id="3760"} 0 +paperless_document_type_document_count{id="5558"} 20 +# HELP paperless_document_type_info Static information about a document type. +# TYPE paperless_document_type_info gauge +paperless_document_type_info{id="3760",name="Contract",slug="contract"} 1 +paperless_document_type_info{id="5558",name="Purchase order",slug="po"} 1 +`) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d0c3d9c --- /dev/null +++ b/go.mod @@ -0,0 +1,42 @@ +module github.com/hansmi/prometheus-paperless-exporter + +go 1.19 + +require ( + github.com/alecthomas/kingpin/v2 v2.3.2 + github.com/go-kit/log v0.2.1 + github.com/google/go-cmp v0.5.9 + github.com/hansmi/paperhooks v0.0.3 + github.com/prometheus/client_golang v1.16.0 + github.com/prometheus/common v0.44.0 + github.com/prometheus/exporter-toolkit v0.10.0 + golang.org/x/sync v0.3.0 +) + +require ( + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/go-resty/resty/v2 v2.7.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/procfs v0.11.0 // indirect + github.com/xhit/go-str2duration/v2 v2.1.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.10.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/oauth2 v0.9.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d157d34 --- /dev/null +++ b/go.sum @@ -0,0 +1,101 @@ +github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU= +github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= +github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/hansmi/paperhooks v0.0.3 h1:0RgPHZaEG790kjtA9iYks+4pxLOkq4UHf2oNquA8aXk= +github.com/hansmi/paperhooks v0.0.3/go.mod h1:OQVL9560CVYa3lLVYPQ0WrRzGMs9ibyY3HSPrLRXEz4= +github.com/jarcoal/httpmock v1.3.0 h1:2RJ8GP0IIaWwcC9Fp2BmVi8Kog3v2Hn7VXM3fTd+nuc= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/exporter-toolkit v0.10.0 h1:yOAzZTi4M22ZzVxD+fhy1URTuNRj/36uQJJ5S8IPza8= +github.com/prometheus/exporter-toolkit v0.10.0/go.mod h1:+sVFzuvV5JDyw+Ih6p3zFxZNVnKQa3x5qPmDSiPu4ZY= +github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk= +github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= +golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/ref/ref.go b/internal/ref/ref.go new file mode 100644 index 0000000..ce629cb --- /dev/null +++ b/internal/ref/ref.go @@ -0,0 +1,8 @@ +package ref + +// Return a pointer to a value. +// +// https://github.com/golang/go/issues/45624 +func Ref[T any](x T) *T { + return &x +} diff --git a/internal/testutil/metrics.go b/internal/testutil/metrics.go new file mode 100644 index 0000000..45bab2e --- /dev/null +++ b/internal/testutil/metrics.go @@ -0,0 +1,34 @@ +package testutil + +import ( + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func CollectAndCompare(t *testing.T, c prometheus.Collector, want string, metricNames ...string) { + t.Helper() + + if err := testutil.CollectAndCompare(c, strings.NewReader(want), metricNames...); err != nil { + t.Error(err) + } +} + +func DiscardMetrics(t *testing.T) chan<- prometheus.Metric { + t.Helper() + + ch := make(chan prometheus.Metric) + + t.Cleanup(func() { + close(ch) + }) + + go func() { + for range ch { + } + }() + + return ch +} diff --git a/log.go b/log.go new file mode 100644 index 0000000..de9865e --- /dev/null +++ b/log.go @@ -0,0 +1,147 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + "runtime" + "strings" + "sync" + "time" + + "github.com/hansmi/paperhooks/pkg/client" + "github.com/prometheus/client_golang/prometheus" + "golang.org/x/sync/errgroup" +) + +type logClient interface { + ListLogs(context.Context) ([]string, *client.Response, error) + GetLog(context.Context, string) ([]client.LogEntry, *client.Response, error) +} + +type logPosition struct { + valid bool + time time.Time + module string + level string +} + +func newLogPosition(e client.LogEntry) logPosition { + return logPosition{ + valid: true, + time: e.Time, + module: e.Module, + level: e.Level, + } +} + +func (p logPosition) equal(e client.LogEntry) bool { + return p.valid && e.Time.Equal(p.time) && e.Module == p.module && e.Level == p.level +} + +type logCollector struct { + cl logClient + + mu sync.Mutex + + seen map[string]logPosition + totalVec *prometheus.CounterVec +} + +func newLogCollector(cl logClient) *logCollector { + return &logCollector{ + cl: cl, + + seen: map[string]logPosition{}, + totalVec: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "paperless_log_entries_total", + Help: `Best-effort count of log entries.`, + }, []string{"name", "module", "level"}), + } +} + +func (c *logCollector) describe(ch chan<- *prometheus.Desc) { + c.totalVec.Describe(ch) +} + +func (c *logCollector) collectOne(ctx context.Context, name string) error { + entries, _, err := c.cl.GetLog(ctx, name) + if err != nil { + var reqErr *client.RequestError + + if errors.As(err, &reqErr) && reqErr.StatusCode == http.StatusNotFound { + return nil + } + + return err + } + + if len(entries) == 0 { + return nil + } + + entryLabels := func(e client.LogEntry) prometheus.Labels { + return prometheus.Labels{ + "name": name, + "module": e.Module, + "level": strings.ToLower(e.Level), + } + } + + c.mu.Lock() + defer c.mu.Unlock() + + start := 0 + + if seen := c.seen[name]; seen.valid { + // Find the first log entry which hasn't been seen previously. + for idx, entry := range entries { + if seen.equal(entry) { + start = idx + 1 + break + } + } + } + + for _, entry := range entries[start:] { + c.totalVec.With(entryLabels(entry)).Inc() + } + + newest := entries[len(entries)-1] + newest.Message = "" + + c.seen[name] = newLogPosition(newest) + + return nil +} + +func (c *logCollector) collect(ctx context.Context, ch chan<- prometheus.Metric) error { + names, _, err := c.cl.ListLogs(ctx) + if err != nil { + return fmt.Errorf("listing log names: %w", err) + } + + g, ctx := errgroup.WithContext(ctx) + g.SetLimit(runtime.GOMAXPROCS(0)) + + for _, name := range names { + name := name + + g.Go(func() error { + if err := c.collectOne(ctx, name); err != nil { + return fmt.Errorf("log %s: %w", name, err) + } + + return nil + }) + } + + if err := g.Wait(); err != nil { + return err + } + + c.totalVec.Collect(ch) + + return nil +} diff --git a/log_test.go b/log_test.go new file mode 100644 index 0000000..594df5d --- /dev/null +++ b/log_test.go @@ -0,0 +1,171 @@ +package main + +import ( + "context" + "errors" + "net/http" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/hansmi/paperhooks/pkg/client" + "github.com/hansmi/prometheus-paperless-exporter/internal/testutil" +) + +type fakeLogClient struct { + names []string + entries map[string][]client.LogEntry + + listErr error + getErr error +} + +func (c *fakeLogClient) addEntries(name string, e []client.LogEntry) { + if c.entries == nil { + c.entries = map[string][]client.LogEntry{} + } + + c.entries[name] = append(c.entries[name], e...) +} + +func (c *fakeLogClient) ListLogs(context.Context) ([]string, *client.Response, error) { + return c.names, nil, c.listErr +} + +func (c *fakeLogClient) GetLog(_ context.Context, name string) ([]client.LogEntry, *client.Response, error) { + if c.getErr != nil { + return nil, nil, c.getErr + } + + entries, ok := c.entries[name] + if !ok { + return nil, nil, &client.RequestError{StatusCode: http.StatusNotFound} + } + + return entries, nil, nil +} + +func TestLog(t *testing.T) { + errTest := errors.New("test error") + + for _, tc := range []struct { + name string + cl fakeLogClient + wantErr error + }{ + {name: "empty"}, + { + name: "listing fails", + cl: fakeLogClient{ + listErr: errTest, + }, + wantErr: errTest, + }, + { + name: "get fails", + cl: fakeLogClient{ + names: []string{"foo", "bar"}, + getErr: errTest, + }, + wantErr: errTest, + }, + { + name: "empty logs", + cl: fakeLogClient{ + names: []string{"first", "second", "error404"}, + entries: map[string][]client.LogEntry{ + "first": nil, + "second": nil, + }, + }, + }, + { + name: "entries", + cl: fakeLogClient{ + names: []string{"first", "second"}, + entries: map[string][]client.LogEntry{ + "first": []client.LogEntry{ + { + Time: time.Date(2020, time.March, 1, 0, 0, 0, 0, time.UTC), + Message: "aaa", + }, + { + Time: time.Date(2020, time.March, 2, 0, 0, 0, 0, time.UTC), + Message: "bbb", + }, + }, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + c := newLogCollector(&tc.cl) + + err := c.collect(context.Background(), testutil.DiscardMetrics(t)) + + if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Error diff (-want +got):\n%s", diff) + } + }) + } +} + +func TestLogCollect(t *testing.T) { + cl := fakeLogClient{ + entries: map[string][]client.LogEntry{}, + } + + c := newMultiCollector(newLogCollector(&cl)) + + testutil.CollectAndCompare(t, c, "") + + cl.names = append(cl.names, "server", "db", "not found") + + testutil.CollectAndCompare(t, c, "") + + cl.addEntries("server", []client.LogEntry{ + { + Time: time.Date(2020, time.March, 2, 0, 0, 0, 0, time.UTC), + Module: "storage", + }, + { + Time: time.Date(2020, time.April, 1, 0, 0, 0, 0, time.UTC), + Module: "storage", + Level: "another", + }, + }) + + testutil.CollectAndCompare(t, c, ` +# HELP paperless_log_entries_total Best-effort count of log entries. +# TYPE paperless_log_entries_total counter +paperless_log_entries_total{level="",module="storage",name="server"} 1 +paperless_log_entries_total{level="another",module="storage",name="server"} 1 +`) + + cl.addEntries("server", []client.LogEntry{ + { + Time: time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC), + Module: "storage", + }, + }) + + for range [3]int{} { + cl.addEntries("db", []client.LogEntry{ + { + Time: time.Date(2021, time.January, 1, 2, 3, 0, 0, time.UTC), + }, + }) + + testutil.CollectAndCompare(t, c, ` +# HELP paperless_log_entries_total Best-effort count of log entries. +# TYPE paperless_log_entries_total counter +paperless_log_entries_total{level="",module="",name="db"} 1 +paperless_log_entries_total{level="",module="storage",name="server"} 2 +paperless_log_entries_total{level="another",module="storage",name="server"} 1 +`) + + // Reset logs + cl.entries = nil + } +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..8e800dc --- /dev/null +++ b/main.go @@ -0,0 +1,65 @@ +package main + +import ( + "io" + "log" + "net/http" + + "github.com/alecthomas/kingpin/v2" + kitlog "github.com/go-kit/log" + "github.com/hansmi/paperhooks/pkg/client" + "github.com/hansmi/paperhooks/pkg/kpflag" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/version" + "github.com/prometheus/exporter-toolkit/web" + webflag "github.com/prometheus/exporter-toolkit/web/kingpinflag" +) + +var webConfig = webflag.AddFlags(kingpin.CommandLine, ":8081") +var metricsPath = kingpin.Flag("web.telemetry-path", "Path under which to expose metrics").Default("/metrics").String() +var disableExporterMetrics = kingpin.Flag("web.disable-exporter-metrics", "Exclude metrics about the exporter itself").Bool() +var timeout = kingpin.Flag("scrape-timeout", "Maximum duration for a scrape").Default("1m").Duration() + +func main() { + var clientFlags client.Flags + + kpflag.RegisterClient(kingpin.CommandLine, &clientFlags) + kingpin.Parse() + + client, err := clientFlags.Build() + if err != nil { + log.Fatal(err) + } + + reg := prometheus.NewPedanticRegistry() + reg.MustRegister(newCollector(client, *timeout)) + + if !*disableExporterMetrics { + reg.MustRegister( + collectors.NewBuildInfoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + collectors.NewGoCollector(), + version.NewCollector("prometheus_paperless_exporter"), + ) + } + + http.Handle(*metricsPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, ` + Paperless Exporter + +

Paperless Exporter

+

Metrics

+ + `) + }) + + logger := kitlog.NewLogfmtLogger(kitlog.StdlibWriter{}) + server := &http.Server{} + + if err := web.ListenAndServe(server, webConfig, logger); err != nil { + log.Fatal(err) + } +} diff --git a/multicollector.go b/multicollector.go new file mode 100644 index 0000000..f3c8276 --- /dev/null +++ b/multicollector.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "runtime" + "time" + + "github.com/prometheus/client_golang/prometheus" + "golang.org/x/sync/errgroup" +) + +type multiCollectorMember interface { + describe(chan<- *prometheus.Desc) + collect(context.Context, chan<- prometheus.Metric) error +} + +type multiCollector struct { + // Impose a timeout on collection if non-zero. + timeout time.Duration + + members []multiCollectorMember +} + +var _ prometheus.Collector = (*multiCollector)(nil) + +func newMultiCollector(m ...multiCollectorMember) *multiCollector { + return &multiCollector{ + members: m, + } +} + +func (c *multiCollector) Describe(ch chan<- *prometheus.Desc) { + for _, i := range c.members { + i.describe(ch) + } +} + +func (c *multiCollector) Collect(ch chan<- prometheus.Metric) { + ctx := context.Background() + + if c.timeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, c.timeout) + defer cancel() + } + + g, ctx := errgroup.WithContext(ctx) + g.SetLimit(runtime.GOMAXPROCS(0)) + + for _, i := range c.members { + collect := i.collect + g.Go(func() error { return collect(ctx, ch) }) + } + + if err := g.Wait(); err != nil { + ch <- prometheus.NewInvalidMetric( + prometheus.NewDesc("paperless_error", "Metrics collection failed", nil, nil), + err) + } +} diff --git a/storagepath.go b/storagepath.go new file mode 100644 index 0000000..d23a4a9 --- /dev/null +++ b/storagepath.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "strconv" + + "github.com/hansmi/paperhooks/pkg/client" + "github.com/prometheus/client_golang/prometheus" +) + +type storagePathClient interface { + ListAllStoragePaths(context.Context, *client.ListStoragePathsOptions, func(context.Context, client.StoragePath) error) error +} + +type storagePathCollector struct { + cl storagePathClient + + infoDesc *prometheus.Desc + docCountDesc *prometheus.Desc +} + +func newStoragePathCollector(cl storagePathClient) *storagePathCollector { + return &storagePathCollector{ + cl: cl, + + infoDesc: prometheus.NewDesc("paperless_storage_path_info", + "Static information about a storage path.", + []string{"id", "name", "slug"}, nil), + docCountDesc: prometheus.NewDesc("paperless_storage_path_document_count", + "Number of documents associated with a storage path.", + []string{"id"}, nil), + } +} + +func (c *storagePathCollector) describe(ch chan<- *prometheus.Desc) { + ch <- c.infoDesc + ch <- c.docCountDesc +} + +func (c *storagePathCollector) collect(ctx context.Context, ch chan<- prometheus.Metric) error { + opts := &client.ListStoragePathsOptions{} + opts.Ordering.Field = "name" + + return c.cl.ListAllStoragePaths(ctx, opts, func(_ context.Context, sp client.StoragePath) error { + id := strconv.FormatInt(sp.ID, 10) + + ch <- prometheus.MustNewConstMetric(c.infoDesc, prometheus.GaugeValue, 1, + id, + sp.Name, + sp.Slug, + ) + + ch <- prometheus.MustNewConstMetric(c.docCountDesc, prometheus.GaugeValue, + float64(sp.DocumentCount), id) + + return nil + }) +} diff --git a/storagepath_test.go b/storagepath_test.go new file mode 100644 index 0000000..40aba06 --- /dev/null +++ b/storagepath_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/hansmi/paperhooks/pkg/client" + "github.com/hansmi/prometheus-paperless-exporter/internal/testutil" +) + +type fakeStoragePathClient struct { + items []client.StoragePath + err error +} + +func (c *fakeStoragePathClient) ListAllStoragePaths(ctx context.Context, opts *client.ListStoragePathsOptions, handler func(context.Context, client.StoragePath) error) error { + for _, i := range c.items { + if err := handler(ctx, i); err != nil { + return err + } + } + + return c.err +} + +func TestStoragePath(t *testing.T) { + errTest := errors.New("test error") + + for _, tc := range []struct { + name string + cl fakeStoragePathClient + wantErr error + }{ + { + name: "empty", + }, + { + name: "listing fails", + cl: fakeStoragePathClient{ + err: errTest, + }, + wantErr: errTest, + }, + { + name: "storagePaths", + cl: fakeStoragePathClient{ + items: []client.StoragePath{ + {ID: 70}, + {ID: 20667}, + {ID: 12805}, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + c := newStoragePathCollector(&tc.cl) + + err := c.collect(context.Background(), testutil.DiscardMetrics(t)) + + if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Error diff (-want +got):\n%s", diff) + } + }) + } +} + +func TestStoragePathCollect(t *testing.T) { + cl := fakeStoragePathClient{} + + c := newMultiCollector(newStoragePathCollector(&cl)) + + testutil.CollectAndCompare(t, c, ``) + + cl.items = append(cl.items, []client.StoragePath{ + {ID: 23547, Name: "personal", Slug: "personal"}, + {ID: 704, Name: "work", DocumentCount: 13}, + }...) + + testutil.CollectAndCompare(t, c, ` +# HELP paperless_storage_path_document_count Number of documents associated with a storage path. +# TYPE paperless_storage_path_document_count gauge +paperless_storage_path_document_count{id="23547"} 0 +paperless_storage_path_document_count{id="704"} 13 +# HELP paperless_storage_path_info Static information about a storage path. +# TYPE paperless_storage_path_info gauge +paperless_storage_path_info{id="23547",name="personal",slug="personal"} 1 +paperless_storage_path_info{id="704",name="work",slug=""} 1 +`) +} diff --git a/tag.go b/tag.go new file mode 100644 index 0000000..6f827e2 --- /dev/null +++ b/tag.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "strconv" + + "github.com/hansmi/paperhooks/pkg/client" + "github.com/prometheus/client_golang/prometheus" +) + +type tagClient interface { + ListAllTags(context.Context, *client.ListTagsOptions, func(context.Context, client.Tag) error) error +} + +type tagCollector struct { + cl tagClient + + infoDesc *prometheus.Desc + docCountDesc *prometheus.Desc +} + +func newTagCollector(cl tagClient) *tagCollector { + return &tagCollector{ + cl: cl, + + infoDesc: prometheus.NewDesc("paperless_tag_info", + "Static information about a tag.", + []string{"id", "name", "slug"}, nil), + docCountDesc: prometheus.NewDesc("paperless_tag_document_count", + "Number of documents associated with a tag.", + []string{"id"}, nil), + } +} + +func (c *tagCollector) describe(ch chan<- *prometheus.Desc) { + ch <- c.infoDesc + ch <- c.docCountDesc +} + +func (c *tagCollector) collect(ctx context.Context, ch chan<- prometheus.Metric) error { + opts := &client.ListTagsOptions{} + opts.Ordering.Field = "name" + + return c.cl.ListAllTags(ctx, opts, func(_ context.Context, tag client.Tag) error { + id := strconv.FormatInt(tag.ID, 10) + + ch <- prometheus.MustNewConstMetric(c.infoDesc, prometheus.GaugeValue, 1, + id, + tag.Name, + tag.Slug, + ) + + ch <- prometheus.MustNewConstMetric(c.docCountDesc, prometheus.GaugeValue, + float64(tag.DocumentCount), id) + + return nil + }) +} diff --git a/tag_test.go b/tag_test.go new file mode 100644 index 0000000..3cfa5d8 --- /dev/null +++ b/tag_test.go @@ -0,0 +1,98 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/hansmi/paperhooks/pkg/client" + "github.com/hansmi/prometheus-paperless-exporter/internal/testutil" +) + +type fakeTagClient struct { + items []client.Tag + err error +} + +func (c *fakeTagClient) ListAllTags(ctx context.Context, opts *client.ListTagsOptions, handler func(context.Context, client.Tag) error) error { + for _, i := range c.items { + if err := handler(ctx, i); err != nil { + return err + } + } + + return c.err +} + +func TestTag(t *testing.T) { + errTest := errors.New("test error") + + for _, tc := range []struct { + name string + cl fakeTagClient + wantErr error + }{ + { + name: "empty", + }, + { + name: "listing fails", + cl: fakeTagClient{ + err: errTest, + }, + wantErr: errTest, + }, + { + name: "tags", + cl: fakeTagClient{ + items: []client.Tag{ + {ID: 8463}, + {ID: 8463}, + {ID: 338}, + {ID: 11768}, + {ID: 30619}, + {ID: 27086}, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTagCollector(&tc.cl) + + err := c.collect(context.Background(), testutil.DiscardMetrics(t)) + + if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Error diff (-want +got):\n%s", diff) + } + }) + } +} + +func TestTagCollect(t *testing.T) { + cl := fakeTagClient{} + + c := newMultiCollector(newTagCollector(&cl)) + + testutil.CollectAndCompare(t, c, ``) + + cl.items = append(cl.items, []client.Tag{ + {ID: 8463, Name: "aaa", Slug: "aslug"}, + {ID: 338, Name: "three-three-eight", DocumentCount: 13}, + {ID: 26429, Name: "last"}, + }...) + + testutil.CollectAndCompare(t, c, ` +# HELP paperless_tag_document_count Number of documents associated with a tag. +# TYPE paperless_tag_document_count gauge +paperless_tag_document_count{id="26429"} 0 +paperless_tag_document_count{id="338"} 13 +paperless_tag_document_count{id="8463"} 0 +# HELP paperless_tag_info Static information about a tag. +# TYPE paperless_tag_info gauge +paperless_tag_info{id="26429",name="last",slug=""} 1 +paperless_tag_info{id="338",name="three-three-eight",slug=""} 1 +paperless_tag_info{id="8463",name="aaa",slug="aslug"} 1 +`) +} diff --git a/task.go b/task.go new file mode 100644 index 0000000..ead2987 --- /dev/null +++ b/task.go @@ -0,0 +1,126 @@ +package main + +import ( + "context" + "strconv" + "strings" + "time" + + "github.com/hansmi/paperhooks/pkg/client" + "github.com/prometheus/client_golang/prometheus" +) + +func optionalTimestamp(t *time.Time) float64 { + if t == nil || t.IsZero() { + return 0 + } + + return float64(t.UnixMilli()) / 1000 +} + +type taskClient interface { + ListTasks(context.Context) ([]client.Task, *client.Response, error) +} + +type taskCollector struct { + cl taskClient + + infoDesc *prometheus.Desc + createdDesc *prometheus.Desc + doneDesc *prometheus.Desc + statusDesc *prometheus.Desc + filenameDesc *prometheus.Desc + + statusInfoVec *prometheus.GaugeVec +} + +func newTaskCollector(cl taskClient) *taskCollector { + c := &taskCollector{ + cl: cl, + + infoDesc: prometheus.NewDesc("paperless_task_info", + "Static information about a task.", + []string{"id", "task_id", "type"}, nil), + createdDesc: prometheus.NewDesc("paperless_task_created_timestamp_seconds", + "Number of seconds since 1970 of the task creation.", + []string{"id"}, nil), + doneDesc: prometheus.NewDesc("paperless_task_done_timestamp_seconds", + "Number of seconds since 1970 of when the task finished.", + []string{"id"}, nil), + statusDesc: prometheus.NewDesc("paperless_task_status", + "Task status.", + []string{"id", "status"}, nil), + filenameDesc: prometheus.NewDesc("paperless_task_filename", + "Filename associated with the task (if any).", + []string{"id", "filename"}, nil), + + statusInfoVec: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "paperless_task_status_info", + Help: "Task status names.", + }, []string{"status"}), + } + + c.ensureStatusInfo(client.TaskSuccess) + + return c +} + +// Returns a canonicalized status string for labels. +func (c *taskCollector) ensureStatusInfo(s client.TaskStatus) string { + status := strings.ToLower(s.String()) + + c.statusInfoVec.With(prometheus.Labels{ + "status": status, + }).Set(1) + + return status +} + +func (c *taskCollector) describe(ch chan<- *prometheus.Desc) { + c.statusInfoVec.Describe(ch) + + ch <- c.infoDesc + ch <- c.createdDesc + ch <- c.doneDesc + ch <- c.statusDesc + ch <- c.filenameDesc +} + +func (c *taskCollector) collect(ctx context.Context, ch chan<- prometheus.Metric) error { + tasks, _, err := c.cl.ListTasks(ctx) + if err != nil { + return err + } + + for _, task := range tasks { + var filename string + + if task.TaskFileName != nil { + filename = *task.TaskFileName + } + + id := strconv.FormatInt(task.ID, 10) + + ch <- prometheus.MustNewConstMetric(c.infoDesc, prometheus.GaugeValue, 1, + id, + task.TaskID, + task.Type, + ) + + ch <- prometheus.MustNewConstMetric(c.createdDesc, prometheus.GaugeValue, + optionalTimestamp(task.Created), id) + + ch <- prometheus.MustNewConstMetric(c.doneDesc, prometheus.GaugeValue, + optionalTimestamp(task.Done), id) + + ch <- prometheus.MustNewConstMetric(c.statusDesc, prometheus.GaugeValue, + 1, id, c.ensureStatusInfo(task.Status)) + + ch <- prometheus.MustNewConstMetric(c.filenameDesc, prometheus.GaugeValue, + 1, id, filename) + } + + c.statusInfoVec.Collect(ch) + + return nil +} diff --git a/task_test.go b/task_test.go new file mode 100644 index 0000000..98ee312 --- /dev/null +++ b/task_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/hansmi/paperhooks/pkg/client" + "github.com/hansmi/prometheus-paperless-exporter/internal/ref" + "github.com/hansmi/prometheus-paperless-exporter/internal/testutil" +) + +type fakeTaskClient struct { + tasks []client.Task + listErr error +} + +func (c *fakeTaskClient) ListTasks(context.Context) ([]client.Task, *client.Response, error) { + return c.tasks, nil, c.listErr +} + +func TestTask(t *testing.T) { + errTest := errors.New("test error") + + for _, tc := range []struct { + name string + cl fakeTaskClient + wantErr error + }{ + {name: "empty"}, + { + name: "listing fails", + cl: fakeTaskClient{ + listErr: errTest, + }, + wantErr: errTest, + }, + { + name: "tasks", + cl: fakeTaskClient{ + tasks: []client.Task{ + {ID: 2942}, + {ID: 27064}, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTaskCollector(&tc.cl) + + err := c.collect(context.Background(), testutil.DiscardMetrics(t)) + + if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Error diff (-want +got):\n%s", diff) + } + }) + } +} + +func TestTaskCollect(t *testing.T) { + cl := fakeTaskClient{} + + c := newMultiCollector(newTaskCollector(&cl)) + + testutil.CollectAndCompare(t, c, ` +# HELP paperless_task_status_info Task status names. +# TYPE paperless_task_status_info gauge +paperless_task_status_info{status="success"} 1 +`) + + cl.tasks = append(cl.tasks, client.Task{ + ID: 31563, + Created: ref.Ref(time.Date(1980, time.January, 1, 0, 0, 0, 0, time.UTC)), + }) + + testutil.CollectAndCompare(t, c, ` +# HELP paperless_task_created_timestamp_seconds Number of seconds since 1970 of the task creation. +# TYPE paperless_task_created_timestamp_seconds gauge +paperless_task_created_timestamp_seconds{id="31563"} 3.155328e+08 +# HELP paperless_task_done_timestamp_seconds Number of seconds since 1970 of when the task finished. +# TYPE paperless_task_done_timestamp_seconds gauge +paperless_task_done_timestamp_seconds{id="31563"} 0 +# HELP paperless_task_filename Filename associated with the task (if any). +# TYPE paperless_task_filename gauge +paperless_task_filename{filename="",id="31563"} 1 +# HELP paperless_task_info Static information about a task. +# TYPE paperless_task_info gauge +paperless_task_info{id="31563",task_id="",type=""} 1 +# HELP paperless_task_status Task status. +# TYPE paperless_task_status gauge +paperless_task_status{id="31563",status="statusunspecified"} 1 +# HELP paperless_task_status_info Task status names. +# TYPE paperless_task_status_info gauge +paperless_task_status_info{status="statusunspecified"} 1 +paperless_task_status_info{status="success"} 1 +`) +}