diff --git a/plugin/docker/docker.go b/plugin/docker/docker.go
index 2c1da761..f3fd9711 100644
--- a/plugin/docker/docker.go
+++ b/plugin/docker/docker.go
@@ -150,7 +150,7 @@ func (p *plugin) findAndCreateUpstream(conn libplugin.ConnMetadata, password str
 	for _, pipe := range pipes {
 
 		// test password
-		if publicKey == nil {
+		if publicKey == nil && password != "" {
 			if pipe.ClientUsername != user {
 				continue
 			}
diff --git a/plugin/yaml/yaml.go b/plugin/yaml/yaml.go
index 2a8205ec..e84b0faf 100644
--- a/plugin/yaml/yaml.go
+++ b/plugin/yaml/yaml.go
@@ -252,7 +252,7 @@ func (p *plugin) findAndCreateUpstream(conn libplugin.ConnMetadata, password str
 				continue
 			}
 
-			if publicKey == nil {
+			if publicKey == nil && password != "" {
 				return p.createUpstream(conn, pipe.To, password)
 			}