Metrics server plugin (#622)
* Metrics server plugin * Link to metrics from root README
This commit is contained in:
Ben Perry
GitHub
parent
ce9b0feaea
commit
3dbb1a1b69
5 changed files with 219 additions and 0 deletions
|
|
@ -81,6 +81,7 @@ Here illustrates the example of `addional challenge` before the `fixed` plugin.
|
|||
|
||||
* 🔀: routing plugin
|
||||
* 🔒: addtional challenge plugin
|
||||
* 📈: metrics plugin
|
||||
|
||||
Plugin list
|
||||
|
||||
|
|
@ -96,6 +97,7 @@ Plugin list
|
|||
* [restful](https://github.com/11notes/docker-sshpiper) by [@11notes](https://github.com/11notes) 🔀🔒: The rest plugin for sshpiperd is a simple plugin that allows you to use a restful backend for authentication and challenge.
|
||||
* [failtoban](plugin/failtoban/) 🔒: ban ip after failed login attempts
|
||||
* [openpubkey](https://github.com/tg123/sshpiper-openpubkey)🔀🔒: integrate with [openpubkey](https://github.com/openpubkey/openpubkey)
|
||||
* [metrics](plugin/metrics/) 📈: serve prometheus metrics on open connections and auth errors
|
||||
|
||||
## Screening recording
|
||||
|
||||
|
|
|
|||
6
go.mod
6
go.mod
|
|
@ -12,6 +12,7 @@ require (
|
|||
github.com/google/uuid v1.6.0
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/pires/go-proxyproto v0.8.1
|
||||
github.com/prometheus/client_golang v1.22.0
|
||||
github.com/ramr/go-reaper v0.3.0
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
github.com/tg123/go-htpasswd v1.2.4
|
||||
|
|
@ -31,7 +32,9 @@ require (
|
|||
|
||||
require (
|
||||
github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
|
|
@ -42,6 +45,9 @@ require (
|
|||
github.com/google/gnostic-models v0.6.9 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/sys/atomicwriter v0.1.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.62.0 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect
|
||||
|
|
|
|||
16
go.sum
16
go.sum
|
|
@ -4,10 +4,14 @@ github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcv
|
|||
github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec=
|
||||
github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
|
||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
|
|
@ -79,6 +83,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
|
|||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
|
|
@ -86,6 +92,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
|||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
|
|
@ -124,6 +132,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
|
|||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs=
|
||||
github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
|
||||
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
|
||||
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
|
||||
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
|
||||
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
|
||||
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
|
||||
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
|
||||
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
|
||||
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
|
||||
github.com/ramr/go-reaper v0.3.0 h1:6E5NsjnAPNeY3vMGBWKZd2D9/eQkpMdefW7w9ybA0KY=
|
||||
github.com/ramr/go-reaper v0.3.0/go.mod h1:bgru3llkYWSj8qb6akpA0sh0pq468OQ5wqvFT3BFHsE=
|
||||
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
|
||||
|
|
|
|||
23
plugin/metrics/README.md
Normal file
23
plugin/metrics/README.md
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
# metrics plugin for sshpiperd
|
||||
|
||||
The metrics plugin provides a server to report metrics on active pipes and connection failures.
|
||||
By default, only the sshpiper_pipe_open_connections{remote_addr="...",user="..."} metric is collected.
|
||||
|
||||
## Metrics
|
||||
|
||||
Name | Type | Labels | Description
|
||||
------------------------------- | ------- | ------------------------- | ------------
|
||||
sshpiper_pipe_open_connections | Gauge | remote_addr, user | Incremented each time a pipe is successfully started, decremented on close
|
||||
sshpiper_pipe_create_errors | Counter | remote_addr | Incremented each time a pipe fails to be created (disabled by default)
|
||||
sshpiper_upstream_auth_failures | Counter | remote_addr, user, method | Incremented each time an upstream rejects the authentication method (disabled by default)
|
||||
|
||||
## Usage
|
||||
|
||||
Note: this is a metrics server plugin. 📈 you must use it with other routing/auth plugins.
|
||||
|
||||
```
|
||||
sshpiperd other-plugin --other-option -- metrics --port <metrics-port>
|
||||
```
|
||||
|
||||
Start the plugin with --collect-pipe-create-errors to enable sshpiper_pipe_create_errors
|
||||
Start the plugin with --collect-upstream-auth-failures to enable sshpiper_upstream_auth_failures
|
||||
172
plugin/metrics/main.go
Normal file
172
plugin/metrics/main.go
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/tg123/sshpiper/libplugin"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
func main() {
|
||||
libplugin.CreateAndRunPluginTemplate(&libplugin.PluginTemplate{
|
||||
Name: "metrics",
|
||||
Usage: "sshpiperd metrics plugin, expose prometheus metrics after login",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "address",
|
||||
Usage: "Metrics server listen address",
|
||||
Required: false,
|
||||
EnvVars: []string{"SSHPIPERD_METRICS_ADDRESS"},
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "port",
|
||||
Usage: "Metrics server listen port",
|
||||
Required: false,
|
||||
Value: 9000,
|
||||
EnvVars: []string{"SSHPIPERD_METRICS_PORT"},
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "collect-pipe-create-errors",
|
||||
Usage: "Collect metrics on pipe creation errors",
|
||||
Required: false,
|
||||
Value: false,
|
||||
EnvVars: []string{"SSHPIPERD_METRICS_COLLECT_PIPE_CREATE_ERRORS"},
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "collect-upstream-auth-failures",
|
||||
Usage: "Collect metrics on upstream auth failures",
|
||||
Required: false,
|
||||
Value: false,
|
||||
EnvVars: []string{"SSHPIPERD_METRICS_COLLECT_UPSTREAM_AUTH_FAILURES"},
|
||||
},
|
||||
},
|
||||
CreateConfig: func(c *cli.Context) (*libplugin.SshPiperPluginConfig, error) {
|
||||
port := c.Int("port")
|
||||
address := c.String("address")
|
||||
bindAddress := fmt.Sprintf("%v:%v", address, port)
|
||||
metrics, config := newPrometheusMetrics(
|
||||
c.Bool("collect-pipe-create-errors"), c.Bool("collect-upstream-auth-failures"),
|
||||
)
|
||||
go func(metrics *prometheusMetrics, bindAddress string) {
|
||||
if err := metrics.ListenAndServe(bindAddress); err != nil {
|
||||
log.Error("Metrics server error:", err)
|
||||
}
|
||||
}(metrics, bindAddress)
|
||||
log.Info("Metrics server is listening on: ", bindAddress)
|
||||
return config, nil
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func newPrometheusMetrics(collectPipeCreateErrors, collectUpstreamAuthFailures bool) (*prometheusMetrics, *libplugin.SshPiperPluginConfig) {
|
||||
registry := prometheus.NewRegistry()
|
||||
openConnections := prometheus.NewGaugeVec(
|
||||
// sshpiper_pipe_open_connections
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "sshpiper",
|
||||
Subsystem: "pipe",
|
||||
Name: "open_connections",
|
||||
Help: "Number of open connections that currently exist partitioned by remote_addr and user",
|
||||
},
|
||||
[]string{"remote_addr", "username"},
|
||||
)
|
||||
registry.MustRegister(openConnections)
|
||||
|
||||
metrics := &prometheusMetrics{
|
||||
registry: registry,
|
||||
openConnections: openConnections,
|
||||
}
|
||||
config := &libplugin.SshPiperPluginConfig{
|
||||
PipeStartCallback: metrics.pipeStartCallback,
|
||||
PipeErrorCallback: metrics.pipeErrorCallback,
|
||||
}
|
||||
|
||||
// Optional metrics
|
||||
if collectPipeCreateErrors {
|
||||
metrics.pipeCreateErrors = prometheus.NewCounterVec(
|
||||
// sshpiper_pipe_create_errors
|
||||
prometheus.CounterOpts{
|
||||
Namespace: "sshpiper",
|
||||
Subsystem: "pipe",
|
||||
Name: "create_errors",
|
||||
Help: "Number of create pipe errors partitioned by remote_addr",
|
||||
},
|
||||
[]string{"remote_addr"},
|
||||
)
|
||||
registry.MustRegister(metrics.pipeCreateErrors)
|
||||
config.PipeCreateErrorCallback = metrics.pipeCreateErrorCallback
|
||||
}
|
||||
if collectUpstreamAuthFailures {
|
||||
metrics.upstreamAuthFailures = prometheus.NewCounterVec(
|
||||
// sshpiper_upstream_auth_failures
|
||||
prometheus.CounterOpts{
|
||||
Namespace: "sshpiper",
|
||||
Subsystem: "upstream",
|
||||
Name: "auth_failures",
|
||||
Help: "Number of upstream auth failures partitioned by remote_addr, user, and method",
|
||||
},
|
||||
[]string{"remote_addr", "user", "method"},
|
||||
)
|
||||
registry.MustRegister(metrics.upstreamAuthFailures)
|
||||
config.UpstreamAuthFailureCallback = metrics.upstreamAuthFailureCallback
|
||||
}
|
||||
return metrics, config
|
||||
}
|
||||
|
||||
type prometheusMetrics struct {
|
||||
registry *prometheus.Registry
|
||||
|
||||
openConnections *prometheus.GaugeVec
|
||||
pipeCreateErrors *prometheus.CounterVec
|
||||
upstreamAuthFailures *prometheus.CounterVec
|
||||
}
|
||||
|
||||
func (ms *prometheusMetrics) ListenAndServe(addr string) error {
|
||||
http.Handle("/metrics", promhttp.InstrumentMetricHandler(
|
||||
ms.registry, promhttp.HandlerFor(ms.registry, promhttp.HandlerOpts{
|
||||
ErrorLog: errorLogger{},
|
||||
}),
|
||||
))
|
||||
return http.ListenAndServe(addr, nil)
|
||||
}
|
||||
|
||||
func (ms *prometheusMetrics) pipeStartCallback(conn libplugin.ConnMetadata) {
|
||||
gauge, err := ms.openConnections.GetMetricWithLabelValues(conn.RemoteAddr(), conn.User())
|
||||
if err != nil {
|
||||
log.Error("Failed to fetch gauge for pipe start callback: ", err)
|
||||
return
|
||||
}
|
||||
gauge.Inc()
|
||||
}
|
||||
|
||||
func (ms *prometheusMetrics) pipeErrorCallback(conn libplugin.ConnMetadata, _ error) {
|
||||
ms.openConnections.DeleteLabelValues(conn.RemoteAddr(), conn.User())
|
||||
}
|
||||
|
||||
func (ms *prometheusMetrics) pipeCreateErrorCallback(remoteAddr string, _ error) {
|
||||
counter, err := ms.pipeCreateErrors.GetMetricWithLabelValues(remoteAddr)
|
||||
if err != nil {
|
||||
log.Error("Failed to get counter for pipe create error callback: ", err)
|
||||
return
|
||||
}
|
||||
counter.Inc()
|
||||
}
|
||||
|
||||
func (ms *prometheusMetrics) upstreamAuthFailureCallback(conn libplugin.ConnMetadata, method string, _ error, _ []string) {
|
||||
counter, err := ms.upstreamAuthFailures.GetMetricWithLabelValues(conn.RemoteAddr(), conn.User(), method)
|
||||
if err != nil {
|
||||
log.Error("Failed to get counter for upstream auth failure callback: ", err)
|
||||
return
|
||||
}
|
||||
counter.Inc()
|
||||
}
|
||||
|
||||
type errorLogger struct{}
|
||||
|
||||
func (l errorLogger) Println(v ...any) {
|
||||
log.Error(v...)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue