Refactor YAML schema to support multiple authorized keys and known hosts (#463)

* add coverage for go proxy

* Refactor CLI flag name for allowed proxy addresses

* Refactor YAML schema to support multiple authorized keys and known hosts

* add testcases

* Refactor YAML schema to support multiple authorized keys and known hosts

* go fmt

* go fmt

e2e/yaml_test.go

@@ -27,7 +27,11 @@ pipes:
   to:
     host: host-password:2222
     username: "user"
-    ignore_hostkey: true
+    known_hosts_data: 
+    # github.com
+    - fDF8RjRwTmVveUZHVEVHcEIyZ3A4RGE0WlE4TGNVPXxycVZYNU0rWTJoS0dteFphcVFBb0syRHp1TEE9IHNzaC1lZDI1NTE5IEFBQUFDM056YUMxbFpESTFOVEU1QUFBQUlPTXFxbmtWenJtMFNkRzZVT29xS0xzYWJnSDVDOW9rV2kwZGgybDlHS0psCg==
+    - {{ .KnownHostsKey }}
+    - {{ .KnownHostsPass }}
 - from:
     - username: "publickey_simple"
       authorized_keys: {{ .AuthorizedKeys_Simple }}
@@ -35,11 +39,13 @@ pipes:
     host: host-publickey:2222
     username: "user"
     private_key: {{ .PrivateKey }}
-    known_hosts_data: {{ .KnownHosts }}
+    known_hosts_data: {{ .KnownHostsKey }}
 - from:
     - username: ".*"
       username_regex_match: true
-      authorized_keys: {{ .AuthorizedKeys_Catchall }}
+      authorized_keys: 
+      - {{ .AuthorizedKeys_Simple }}
+      - {{ .AuthorizedKeys_Catchall }}
   to:
     host: host-publickey:2222
     username: "user"
@@ -114,7 +120,7 @@ func TestYaml(t *testing.T) {
 		}
 	}
 
-	knownHostsData, err := runAndGetStdout(
+	knownHostsKeyData, err := runAndGetStdout(
 		"ssh-keyscan",
 		"-p",
 		"2222",
@@ -125,14 +131,26 @@ func TestYaml(t *testing.T) {
 		t.Errorf("failed to run ssh-keyscan: %v", err)
 	}
 
+	knownHostsPassData, err := runAndGetStdout(
+		"ssh-keyscan",
+		"-p",
+		"2222",
+		"host-password",
+	)
+
+	if err != nil {
+		t.Errorf("failed to run ssh-keyscan : %v", err)
+	}
 	if err := template.Must(template.New("yaml").Parse(yamlConfigTemplate)).ExecuteTemplate(yamlfile, "yaml", struct {
-		KnownHosts string
+		KnownHostsKey  string
+		KnownHostsPass string
 		PrivateKey     string
 
 		AuthorizedKeys_Simple   string
 		AuthorizedKeys_Catchall string
 	}{
-		KnownHosts: base64.StdEncoding.EncodeToString(knownHostsData),
+		KnownHostsKey:  base64.StdEncoding.EncodeToString(knownHostsKeyData),
+		KnownHostsPass: base64.StdEncoding.EncodeToString(knownHostsPassData),
 		PrivateKey:     path.Join(yamldir, "id_rsa"),
 
 		AuthorizedKeys_Simple:   path.Join(yamldir, "id_rsa_simple.pub"),

plugin/yaml/schema.json

@@ -51,12 +51,32 @@
                     "type": "boolean"
                 },
                 "authorized_keys": {
+                    "oneOf": [
+                        {
+                            "type": "array",
+                            "items": {
                                 "type": "string"
+                            }
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
                 },
                 "authorized_keys_data": {
+                    "oneOf": [
+                        {
+                            "type": "array",
+                            "items": {
                                 "type": "string"
                             }
                         },
+                        {
+                            "type": "string"
+                        }
+                    ]
+                }
+            },
             "required": [
                 "username"
             ]
@@ -84,12 +104,32 @@
                     "type": "string"
                 },
                 "known_hosts": {
+                    "oneOf": [
+                        {
+                            "type": "array",
+                            "items": {
                                 "type": "string"
+                            }
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
                 },
                 "known_hosts_data": {
+                    "oneOf": [
+                        {
+                            "type": "array",
+                            "items": {
                                 "type": "string"
                             }
                         },
+                        {
+                            "type": "string"
+                        }
+                    ]
+                }
+            },
             "required": [
                 "host"
             ]

plugin/yaml/yaml.go

@@ -20,8 +20,8 @@ import (
 type pipeConfigFrom struct {
 	Username           string       `yaml:"username"`
 	UsernameRegexMatch bool         `yaml:"username_regex_match,omitempty"`
-	AuthorizedKeys     string `yaml:"authorized_keys,omitempty"`
+	AuthorizedKeys     listOrString `yaml:"authorized_keys,omitempty"`
-	AuthorizedKeysData string `yaml:"authorized_keys_data,omitempty"`
+	AuthorizedKeysData listOrString `yaml:"authorized_keys_data,omitempty"`
 }
 
 type pipeConfigTo struct {
@@ -30,11 +30,43 @@ type pipeConfigTo struct {
 	Password       string       `yaml:"password,omitempty"`
 	PrivateKey     string       `yaml:"private_key,omitempty"`
 	PrivateKeyData string       `yaml:"private_key_data,omitempty"`
-	KnownHosts     string `yaml:"known_hosts,omitempty"`
+	KnownHosts     listOrString `yaml:"known_hosts,omitempty"`
-	KnownHostsData string `yaml:"known_hosts_data,omitempty"`
+	KnownHostsData listOrString `yaml:"known_hosts_data,omitempty"`
 	IgnoreHostkey  bool         `yaml:"ignore_hostkey,omitempty"`
 }
 
+type listOrString struct {
+	List []string
+	Str  string
+}
+
+func (l *listOrString) Any() bool {
+	return len(l.List) > 0 || l.Str != ""
+}
+
+func (l *listOrString) Combine() []string {
+	if l.Str != "" {
+		return append(l.List, l.Str)
+	}
+	return l.List
+}
+
+func (l *listOrString) UnmarshalYAML(value *yaml.Node) error {
+	// Try to unmarshal as a list
+	var list []string
+	if err := value.Decode(&list); err == nil {
+		l.List = list
+		return nil
+	}
+	// Try to unmarshal as a string
+	var str string
+	if err := value.Decode(&str); err == nil {
+		l.Str = str
+		return nil
+	}
+	return fmt.Errorf("Failed to unmarshal OneOfType")
+}
+
 type pipeConfig struct {
 	From []pipeConfigFrom `yaml:"from,flow"`
 	To   pipeConfigTo     `yaml:"to,flow"`
@@ -129,6 +161,34 @@ func (p *plugin) loadFileOrDecode(file string, base64data string, vars map[strin
 	return nil, nil
 }
 
+func (p *plugin) loadFileOrDecodeMany(files listOrString, base64data listOrString, vars map[string]string) ([]byte, error) {
+	var byteSlices [][]byte
+
+	for _, file := range files.Combine() {
+		data, err := p.loadFileOrDecode(file, "", vars)
+		if err != nil {
+			return nil, err
+		}
+
+		if data != nil {
+			byteSlices = append(byteSlices, data)
+		}
+	}
+
+	for _, data := range base64data.Combine() {
+		decoded, err := base64.StdEncoding.DecodeString(data)
+		if err != nil {
+			return nil, err
+		}
+
+		if decoded != nil {
+			byteSlices = append(byteSlices, decoded)
+		}
+	}
+
+	return bytes.Join(byteSlices, []byte("\n")), nil
+}
+
 func (p *plugin) supportedMethods() ([]string, error) {
 	config, err := p.loadConfig()
 	if err != nil {
@@ -139,7 +199,7 @@ func (p *plugin) supportedMethods() ([]string, error) {
 
 	for _, pipe := range config.Pipes {
 		for _, from := range pipe.From {
-			if from.AuthorizedKeys != "" || from.AuthorizedKeysData != "" {
+			if from.AuthorizedKeys.Any() || from.AuthorizedKeysData.Any() {
 				set["publickey"] = true // found authorized_keys, so we support publickey
 			} else {
 				set["password"] = true // no authorized_keys, so we support password
@@ -163,7 +223,7 @@ func (p *plugin) verifyHostKey(conn libplugin.ConnMetadata, hostname, netaddr st
 
 	to := item.(*pipeConfigTo)
 
-	data, err := p.loadFileOrDecode(to.KnownHosts, to.KnownHostsData, map[string]string{
+	data, err := p.loadFileOrDecodeMany(to.KnownHosts, to.KnownHostsData, map[string]string{
 		"DOWNSTREAM_USER": conn.User(),
 		"UPSTREAM_USER":   to.Username,
 	})
@@ -242,7 +302,7 @@ func (p *plugin) findAndCreateUpstream(conn libplugin.ConnMetadata, password str
 				return p.createUpstream(conn, pipe.To, password)
 			}
 
-			rest, err := p.loadFileOrDecode(from.AuthorizedKeys, from.AuthorizedKeysData, map[string]string{
+			rest, err := p.loadFileOrDecodeMany(from.AuthorizedKeys, from.AuthorizedKeysData, map[string]string{
 				"DOWNSTREAM_USER": user,
 			})
 			if err != nil {

plugin/yaml/yaml_test.go

@@ -0,0 +1,58 @@
+//go:build full || e2e
+
+package main
+
+import (
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+const yamlConfigTemplate = `
+version: "1.0"
+pipes:
+- from:
+    - username: "password_simple"
+  to:
+    host: host-password:2222
+    username: "user"
+    ignore_hostkey: true
+- from:
+    - username: "password_.*_regex"
+      username_regex_match: true
+  to:
+    host: host-password:2222
+    username: "user"
+    known_hosts_data: 
+    - fDF8RjRwTmVveUZHVEVHcEIyZ3A4RGE0WlE4TGNVPXxycVZYNU0rWTJoS0dteFphcVFBb0syRHp1TEE9IHNzaC1lZDI1NTE5IEFBQUFDM056YUMxbFpESTFOVEU1QUFBQUlPTXFxbmtWenJtMFNkRzZVT29xS0xzYWJnSDVDOW9rV2kwZGgybDlHS0psCg==
+    - fDF8VzRpUUd0VFVyREJwSjM3RnFuOWRwcEdVRE5jPXxEZWFna2RwVHpZZDExdDhYWXlORnlhZmROZ2c9IHNzaC1lZDI1NTE5IEFBQUFDM056YUMxbFpESTFOVEU1QUFBQUlBZnVDSEtWVGpxdXh2dDZDTTZ0ZEc0U0xwMUJ0bi9uT2VISEU1VU96UmRmCg==
+- from:
+    - username: "publickey_simple"
+      authorized_keys: /tmp/auth_keys
+  to:
+    host: host-publickey:2222
+    username: "user"
+    private_key: /tmp/private_key
+    known_hosts_data: fDF8RjRwTmVveUZHVEVHcEIyZ3A4RGE0WlE4TGNVPXxycVZYNU0rWTJoS0dteFphcVFBb0syRHp1TEE9IHNzaC1lZDI1NTE5IEFBQUFDM056YUMxbFpESTFOVEU1QUFBQUlPTXFxbmtWenJtMFNkRzZVT29xS0xzYWJnSDVDOW9rV2kwZGgybDlHS0psCg==
+- from:
+    - username: ".*"
+      username_regex_match: true
+      authorized_keys: 
+      - /tmp/private_key1
+      - /tmp/private_key2
+  to:
+    host: host-publickey:2222
+    username: "user"
+    ignore_hostkey: true
+    private_key: /tmp/private_key
+`
+
+func TestYamlDecode(t *testing.T) {
+	var config piperConfig
+
+	err := yaml.Unmarshal([]byte(yamlConfigTemplate), &config)
+	if err != nil {
+		t.Fatalf("Failed to unmarshal yaml: %v", err)
+	}
+
+}