diff --git a/README.md b/README.md index 88fbf30c..6dfa6c9a 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,7 @@ Here illustrates the example of `addional challenge` before the `fixed` plugin. * 🔀: routing plugin * 🔒: addtional challenge plugin + * 📈: metrics plugin Plugin list @@ -96,6 +97,7 @@ Plugin list * [restful](https://github.com/11notes/docker-sshpiper) by [@11notes](https://github.com/11notes) 🔀🔒: The rest plugin for sshpiperd is a simple plugin that allows you to use a restful backend for authentication and challenge. * [failtoban](plugin/failtoban/) 🔒: ban ip after failed login attempts * [openpubkey](https://github.com/tg123/sshpiper-openpubkey)🔀🔒: integrate with [openpubkey](https://github.com/openpubkey/openpubkey) + * [metrics](plugin/metrics/) 📈: serve prometheus metrics on open connections and auth errors ## Screening recording diff --git a/go.mod b/go.mod index 472288c5..9486480c 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/google/uuid v1.6.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pires/go-proxyproto v0.8.1 + github.com/prometheus/client_golang v1.22.0 github.com/ramr/go-reaper v0.3.0 github.com/sirupsen/logrus v1.9.3 github.com/tg123/go-htpasswd v1.2.4 @@ -31,7 +32,9 @@ require ( require ( github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -42,6 +45,9 @@ require ( github.com/google/gnostic-models v0.6.9 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect diff --git a/go.sum b/go.sum index added386..3046ed1a 100644 --- a/go.sum +++ b/go.sum @@ -4,10 +4,14 @@ github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcv github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= @@ -79,6 +83,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -86,6 +92,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -124,6 +132,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/ramr/go-reaper v0.3.0 h1:6E5NsjnAPNeY3vMGBWKZd2D9/eQkpMdefW7w9ybA0KY= github.com/ramr/go-reaper v0.3.0/go.mod h1:bgru3llkYWSj8qb6akpA0sh0pq468OQ5wqvFT3BFHsE= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= diff --git a/plugin/metrics/README.md b/plugin/metrics/README.md new file mode 100644 index 00000000..cfb7af6c --- /dev/null +++ b/plugin/metrics/README.md @@ -0,0 +1,23 @@ +# metrics plugin for sshpiperd + +The metrics plugin provides a server to report metrics on active pipes and connection failures. +By default, only the sshpiper_pipe_open_connections{remote_addr="...",user="..."} metric is collected. + +## Metrics + +Name | Type | Labels | Description +------------------------------- | ------- | ------------------------- | ------------ +sshpiper_pipe_open_connections | Gauge | remote_addr, user | Incremented each time a pipe is successfully started, decremented on close +sshpiper_pipe_create_errors | Counter | remote_addr | Incremented each time a pipe fails to be created (disabled by default) +sshpiper_upstream_auth_failures | Counter | remote_addr, user, method | Incremented each time an upstream rejects the authentication method (disabled by default) + +## Usage + +Note: this is a metrics server plugin. 📈 you must use it with other routing/auth plugins. + +``` +sshpiperd other-plugin --other-option -- metrics --port +``` + +Start the plugin with --collect-pipe-create-errors to enable sshpiper_pipe_create_errors +Start the plugin with --collect-upstream-auth-failures to enable sshpiper_upstream_auth_failures diff --git a/plugin/metrics/main.go b/plugin/metrics/main.go new file mode 100644 index 00000000..97d0cda9 --- /dev/null +++ b/plugin/metrics/main.go @@ -0,0 +1,172 @@ +package main + +import ( + "fmt" + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + log "github.com/sirupsen/logrus" + "github.com/tg123/sshpiper/libplugin" + "github.com/urfave/cli/v2" +) + +func main() { + libplugin.CreateAndRunPluginTemplate(&libplugin.PluginTemplate{ + Name: "metrics", + Usage: "sshpiperd metrics plugin, expose prometheus metrics after login", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "address", + Usage: "Metrics server listen address", + Required: false, + EnvVars: []string{"SSHPIPERD_METRICS_ADDRESS"}, + }, + &cli.IntFlag{ + Name: "port", + Usage: "Metrics server listen port", + Required: false, + Value: 9000, + EnvVars: []string{"SSHPIPERD_METRICS_PORT"}, + }, + &cli.BoolFlag{ + Name: "collect-pipe-create-errors", + Usage: "Collect metrics on pipe creation errors", + Required: false, + Value: false, + EnvVars: []string{"SSHPIPERD_METRICS_COLLECT_PIPE_CREATE_ERRORS"}, + }, + &cli.BoolFlag{ + Name: "collect-upstream-auth-failures", + Usage: "Collect metrics on upstream auth failures", + Required: false, + Value: false, + EnvVars: []string{"SSHPIPERD_METRICS_COLLECT_UPSTREAM_AUTH_FAILURES"}, + }, + }, + CreateConfig: func(c *cli.Context) (*libplugin.SshPiperPluginConfig, error) { + port := c.Int("port") + address := c.String("address") + bindAddress := fmt.Sprintf("%v:%v", address, port) + metrics, config := newPrometheusMetrics( + c.Bool("collect-pipe-create-errors"), c.Bool("collect-upstream-auth-failures"), + ) + go func(metrics *prometheusMetrics, bindAddress string) { + if err := metrics.ListenAndServe(bindAddress); err != nil { + log.Error("Metrics server error:", err) + } + }(metrics, bindAddress) + log.Info("Metrics server is listening on: ", bindAddress) + return config, nil + }, + }) +} + +func newPrometheusMetrics(collectPipeCreateErrors, collectUpstreamAuthFailures bool) (*prometheusMetrics, *libplugin.SshPiperPluginConfig) { + registry := prometheus.NewRegistry() + openConnections := prometheus.NewGaugeVec( + // sshpiper_pipe_open_connections + prometheus.GaugeOpts{ + Namespace: "sshpiper", + Subsystem: "pipe", + Name: "open_connections", + Help: "Number of open connections that currently exist partitioned by remote_addr and user", + }, + []string{"remote_addr", "username"}, + ) + registry.MustRegister(openConnections) + + metrics := &prometheusMetrics{ + registry: registry, + openConnections: openConnections, + } + config := &libplugin.SshPiperPluginConfig{ + PipeStartCallback: metrics.pipeStartCallback, + PipeErrorCallback: metrics.pipeErrorCallback, + } + + // Optional metrics + if collectPipeCreateErrors { + metrics.pipeCreateErrors = prometheus.NewCounterVec( + // sshpiper_pipe_create_errors + prometheus.CounterOpts{ + Namespace: "sshpiper", + Subsystem: "pipe", + Name: "create_errors", + Help: "Number of create pipe errors partitioned by remote_addr", + }, + []string{"remote_addr"}, + ) + registry.MustRegister(metrics.pipeCreateErrors) + config.PipeCreateErrorCallback = metrics.pipeCreateErrorCallback + } + if collectUpstreamAuthFailures { + metrics.upstreamAuthFailures = prometheus.NewCounterVec( + // sshpiper_upstream_auth_failures + prometheus.CounterOpts{ + Namespace: "sshpiper", + Subsystem: "upstream", + Name: "auth_failures", + Help: "Number of upstream auth failures partitioned by remote_addr, user, and method", + }, + []string{"remote_addr", "user", "method"}, + ) + registry.MustRegister(metrics.upstreamAuthFailures) + config.UpstreamAuthFailureCallback = metrics.upstreamAuthFailureCallback + } + return metrics, config +} + +type prometheusMetrics struct { + registry *prometheus.Registry + + openConnections *prometheus.GaugeVec + pipeCreateErrors *prometheus.CounterVec + upstreamAuthFailures *prometheus.CounterVec +} + +func (ms *prometheusMetrics) ListenAndServe(addr string) error { + http.Handle("/metrics", promhttp.InstrumentMetricHandler( + ms.registry, promhttp.HandlerFor(ms.registry, promhttp.HandlerOpts{ + ErrorLog: errorLogger{}, + }), + )) + return http.ListenAndServe(addr, nil) +} + +func (ms *prometheusMetrics) pipeStartCallback(conn libplugin.ConnMetadata) { + gauge, err := ms.openConnections.GetMetricWithLabelValues(conn.RemoteAddr(), conn.User()) + if err != nil { + log.Error("Failed to fetch gauge for pipe start callback: ", err) + return + } + gauge.Inc() +} + +func (ms *prometheusMetrics) pipeErrorCallback(conn libplugin.ConnMetadata, _ error) { + ms.openConnections.DeleteLabelValues(conn.RemoteAddr(), conn.User()) +} + +func (ms *prometheusMetrics) pipeCreateErrorCallback(remoteAddr string, _ error) { + counter, err := ms.pipeCreateErrors.GetMetricWithLabelValues(remoteAddr) + if err != nil { + log.Error("Failed to get counter for pipe create error callback: ", err) + return + } + counter.Inc() +} + +func (ms *prometheusMetrics) upstreamAuthFailureCallback(conn libplugin.ConnMetadata, method string, _ error, _ []string) { + counter, err := ms.upstreamAuthFailures.GetMetricWithLabelValues(conn.RemoteAddr(), conn.User(), method) + if err != nil { + log.Error("Failed to get counter for upstream auth failure callback: ", err) + return + } + counter.Inc() +} + +type errorLogger struct{} + +func (l errorLogger) Println(v ...any) { + log.Error(v...) +}