Skip to content

Commit

Permalink
Add SSH Proxy (#72)
Browse files Browse the repository at this point in the history
- Create additional ResourceAccess and Authentication structures
   and related yaml to store in the database.
- The only supported driver is VMX.  All other drivers need to be
   taught about the new Authentication struct (in the future).
- Add ProxySshAddress configuration variable to fish Config, the
   default value is `0.0.0.0:2022`.
- Add corresponding SSH examples.
- **NOTE**: while `scp` succeeds, it hangs and must be ctrl+C'd.
   This is not getting fixed in this PR.

This feature currently only supports SSH via username/password.
See the new example label creation:

```
authentication:
  username: packer
  password: packer
```

When the fish node is running, you must query the application
resource UID and request `/access` (see new run application
example).  These passwords are **one time use only**, in order
to get a new password, re-request `/access`.
  • Loading branch information
svenevs authored Jul 11, 2024
1 parent e8463e2 commit fb07ac5
Show file tree
Hide file tree
Showing 15 changed files with 695 additions and 14 deletions.
28 changes: 22 additions & 6 deletions cmd/fish/fish.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,16 @@ import (
"github.com/adobe/aquarium-fish/lib/fish"
"github.com/adobe/aquarium-fish/lib/log"
"github.com/adobe/aquarium-fish/lib/openapi"
"github.com/adobe/aquarium-fish/lib/proxy"
"github.com/adobe/aquarium-fish/lib/proxy_socks"
"github.com/adobe/aquarium-fish/lib/proxy_ssh"
)

func main() {
log.Infof("Aquarium Fish %s (%s)", build.Version, build.Time)

var api_address string
var proxy_address string
var proxy_socks_address string
var proxy_ssh_address string
var node_address string
var cluster_join *[]string
var cfg_path string
Expand Down Expand Up @@ -65,8 +67,11 @@ func main() {
if api_address != "" {
cfg.APIAddress = api_address
}
if proxy_address != "" {
cfg.ProxyAddress = proxy_address
if proxy_socks_address != "" {
cfg.ProxySocksAddress = proxy_socks_address
}
if proxy_ssh_address != "" {
cfg.ProxySshAddress = proxy_ssh_address
}
if node_address != "" {
cfg.NodeAddress = node_address
Expand Down Expand Up @@ -125,7 +130,17 @@ func main() {
}

log.Info("Fish starting socks5 proxy...")
err = proxy.Init(fish, cfg.ProxyAddress)
err = proxy_socks.Init(fish, cfg.ProxySocksAddress)
if err != nil {
return err
}

log.Info("Fish starting ssh proxy...")
id_rsa_path := cfg.NodeSSHKey
if !filepath.IsAbs(id_rsa_path) {
id_rsa_path = filepath.Join(cfg.Directory, id_rsa_path)
}
err = proxy_ssh.Init(fish, id_rsa_path, cfg.ProxySshAddress)
if err != nil {
return err
}
Expand Down Expand Up @@ -166,7 +181,8 @@ func main() {

flags := cmd.Flags()
flags.StringVarP(&api_address, "api", "a", "", "address used to expose the fish API")
flags.StringVarP(&proxy_address, "proxy", "p", "", "address used to expose the SOCKS5 proxy")
flags.StringVar(&proxy_socks_address, "socks_proxy", "", "address used to expose the SOCKS5 proxy")
flags.StringVar(&proxy_ssh_address, "ssh_proxy", "", "address used to expose the SSH proxy")
flags.StringVarP(&node_address, "node", "n", "", "node external endpoint to connect to tell the other nodes")
cluster_join = flags.StringSliceP("join", "j", nil, "addresses of existing cluster nodes to join, comma separated")
flags.StringVarP(&cfg_path, "cfg", "c", "", "yaml configuration file")
Expand Down
92 changes: 92 additions & 0 deletions docs/openapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,35 @@ paths:
security:
- basic_auth: []

/api/v1/resource/{uid}/access:
get:
summary: Get SSH access credentials by Resource UID
description: Display any known SSH access information for the Resource
operationId: ResourceAccessPut
tags:
- ResourceAccess
parameters:
- name: uid
in: path
description: UID of the object
required: true
schema:
type: string
format: uuid
responses:
'200':
description: Successful operation
content:
application/json:
schema:
$ref: '#/components/schemas/ResourceAccess'
'401':
$ref: '#/components/responses/UnauthorizedError'
'404':
description: Resource not found
security:
- basic_auth: []

/api/v1/application/:
get:
summary: Get list of Applications
Expand Down Expand Up @@ -1267,6 +1296,9 @@ components:
options:
x-go-type: util.UnparsedJson
description: Driver-specific options to execute the environment
authentication:
$ref: '#/components/schemas/Authentication'
description: Authentication information to connect.
Label:
type: object
description: >
Expand Down Expand Up @@ -1569,6 +1601,66 @@ components:
JENKINS_AGENT_SECRET: 03839eabcf945b1e780be8f9488d264c4c57bf388546da9a84588345555f29b0
JENKINS_AGENT_NAME: test-node
JENKINS_AGENT_WORKSPACE: /Volumes/xcode122
authentication:
$ref: '#/components/schemas/Authentication'
description: Authentication information to connect.

ResourceAccessUID:
type: string
format: uuid
x-oapi-codegen-extra-tags:
gorm: primaryKey
ResourceAccess:
type: object
description: >
An accessor entry to be able to identify and look up different
(currently running) resources.
Used to enable SSH pass-through.
required:
- UID
- created_at
- resource_UID
- username
- password
properties:
UID:
$ref: '#/components/schemas/ResourceAccessUID'
x-oapi-codegen-extra-tags:
gorm: primaryKey
created_at:
x-go-type: time.Time
resource_UID:
# TODO: in OAPI v3.1.0 siblings: $ref: '#/components/schemas/ResourceAccessUID'
type: string
format: uuid
x-oapi-codegen-extra-tags:
yaml: resource_UID
username:
type: string
description: |
The username to use when logging into the fish node.
password:
type: string
description: >
The password to use when logging into the fish node.
Authentication:
type: object
description: >
Authentication information to enable connecting to the machine.
required:
- username
- password
properties:
username:
type: string
description: |
The username to use when logging into Resource.
password:
type: string
description: >
The password to use when loggin into the Resource.
VoteUID:
type: string
Expand Down
54 changes: 54 additions & 0 deletions examples/create_label-vmx-ubuntu2004arm-ci-ssh.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/bin/sh -e
# Copyright 2024 Adobe. All rights reserved.
# This file is licensed to you under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. You may obtain a copy
# of the License at http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
# OF ANY KIND, either express or implied. See the License for the specific language
# governing permissions and limitations under the License.

#
# This example script allows to see the existing Label and create a new version of it
# Please check the images URLs in Label definitions below
#

token=$1
[ "$token" ] || exit 1
hostport=$2
[ "$hostport" ] || hostport=localhost:8001

label=ubuntu2004arm-ci_vmx

# It's a bit dirty, but works for now - probably better to create API call to find the latest label
curr_label=$(curl -s -u "admin:$token" -k "https://$hostport/api/v1/label/?filter=name=\"$label\"" | sed 's/},{/},\n{/g' | tail -1)
curr_version="$(echo "$curr_label" | grep -o '"version": *[0-9]\+' | tr -dc '0-9')"
echo "Current label '$label:$curr_version': $curr_label"

[ "x$curr_version" != "x" ] || curr_version=0
new_version=$(($curr_version+1))

echo
echo "Create the new version of Label '$label:$new_version' ?"
echo "Press any key to create or Ctrl-C to abort"
read w1

label_id=$(curl -s -u "admin:$token" -k -X POST -H 'Content-Type: application/yaml' -d '---
name: "'$label'"
version: '$new_version'
definitions:
- driver: vmx
options:
images: # For test purposes images are used as symlink to aquarium-bait/out so does not need checksum
- url: https://artifact-storage/aquarium/image/vmx/ubuntu2004arm-VERSION/ubuntu2004arm-VERSION.tar.xz
- url: https://artifact-storage/aquarium/image/vmx/ubuntu2004arm-ci-VERSION/ubuntu2004arm-ci-VERSION.tar.xz
resources:
cpu: 4
ram: 4
authentication:
username: packer
password: packer
' "https://$hostport/api/v1/label/" | grep -o '"UID": *"[^"]\+"' | cut -d':' -f 2 | tr -d ' "')

echo "Created Label ID: ${label_id}"
69 changes: 69 additions & 0 deletions examples/run_application-vmx-ubuntu2004arm-ci-ssh.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
#!/bin/sh -e
# Copyright 2024 Adobe. All rights reserved.
# This file is licensed to you under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. You may obtain a copy
# of the License at http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
# OF ANY KIND, either express or implied. See the License for the specific language
# governing permissions and limitations under the License.

#
# This script creates the new Application to allocate resource of the latest version of Label
# Please check the Application metadata below - it defines the jenkins node to connect
#

token=$1
[ "$token" ] || exit 1
hostport=$2
[ "$hostport" ] || hostport=localhost:8001

label=ubuntu2004arm-ci_vmx

# It's a bit dirty, but works for now - probably better to create API call to find the latest label
curr_label=$(curl -s -u "admin:$token" -k "https://$hostport/api/v1/label/?filter=name=\"$label\"" | sed 's/},{"UID":/},\n{"UID":/g' | tail -1)
curr_label_id="$(echo "$curr_label" | grep -o '"UID": *"[^"]\+"' | cut -d':' -f 2 | tr -d ' "')"
if [ "x$curr_label_id" = "x" ]; then
echo "ERROR: Unable to find label '$label' - please create one before running the application"
exit 1
fi

echo "Found label '$label': $curr_label_id : $curr_label"

echo
echo "Press key to create the Application with label '$label'"
read w1

app=$(curl -s -u "admin:$token" -k -X POST -H 'Content-Type: application/yaml' -d '---
label_UID: '$curr_label_id'
metadata:
JENKINS_URL: https://jenkins-host.local/
JENKINS_AGENT_SECRET: 03839eabcf945b1e780be8f9488d264c4c57bf388546da9a84588345555f29b0
JENKINS_AGENT_NAME: test-node
' "https://$hostport/api/v1/application/")
app_id="$(echo "$app" | grep -o '"UID": *"[^"]\+"' | cut -d':' -f 2 | tr -d ' "')"

echo "Application created: $app_id : $app"

echo "Press key to check the application resource"
read w1

response="$(curl -s -u "admin:$token" -k "https://$hostport/api/v1/application/$app_id/resource")"
resource_UID="$(echo "$response" | grep -o '"UID": *"[^"]\+"' | cut -d':' -f 2 | tr -d ' "')"
echo "Application resource:"
echo "$response"
echo "Resource UID: $resource_UID"

echo "Press key to query SSH authentication information"
echo 'You will need to `ssh -p PORT [email protected]`, where PORT by default is 2022'
read w1

# Passwords are one-time use, after it has been used you must re-issue this
# curl command to get a new password.
curl -u "admin:$token" -k "https://$hostport/api/v1/resource/$resource_UID/access"

echo "Press key to deallocate the application resource"
read w1

curl -u "admin:$token" -k "https://$hostport/api/v1/application/$app_id/deallocate"
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
Expand Down
7 changes: 5 additions & 2 deletions lib/drivers/vmx/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -223,8 +223,11 @@ func (d *Driver) Allocate(def types.LabelDefinition, metadata map[string]any) (*
}

log.Info("VMX: Allocate of VM completed:", vmx_path)

return &types.Resource{Identifier: vmx_path, HwAddr: vm_hwaddr}, nil
return &types.Resource{
Identifier: vmx_path,
HwAddr: vm_hwaddr,
Authentication: def.Authentication,
}, nil
}

func (d *Driver) Status(res *types.Resource) (string, error) {
Expand Down
18 changes: 13 additions & 5 deletions lib/fish/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,11 @@ import (
type Config struct {
Directory string `json:"directory"` // Where to store database and other useful data (if relative - to CWD)

APIAddress string `json:"api_address"` // Where to serve Web UI, API & Meta API
ProxyAddress string `json:"proxy_address"` // Where to serve SOCKS5 proxy for the allocated resources
NodeAddress string `json:"node_address"` // What is the external address of the node
ClusterJoin []string `json:"cluster_join"` // The node addresses to join the cluster
APIAddress string `json:"api_address"` // Where to serve Web UI, API & Meta API
ProxySocksAddress string `json:"proxy_socks_address"` // Where to serve SOCKS5 proxy for the allocated resources
ProxySshAddress string `json:"proxy_ssh_address"` // Where to serve SSH proxy for the allocated resources
NodeAddress string `json:"node_address"` // What is the external address of the node
ClusterJoin []string `json:"cluster_join"` // The node addresses to join the cluster

TLSKey string `json:"tls_key"` // TLS PEM private key (if relative - to directory)
TLSCrt string `json:"tls_crt"` // TLS PEM public certificate (if relative - to directory)
Expand All @@ -37,6 +38,8 @@ type Config struct {
NodeLocation string `json:"node_location"` // Specify cluster node location for multi-dc configurations
NodeIdentifiers []string `json:"node_identifiers"` // The list of node identifiers which could be used to find the right Node for Resource

NodeSSHKey string `json:"ssh_key"` // The SSH RSA identity private key for the fish node (if relative - to directory)

DefaultResourceLifetime string `json:"default_resource_lifetime"` // Sets the lifetime of the resource which will be used if label definition one is not set

// Configuration for the node drivers, if defined - only the listed plugins will be loaded
Expand Down Expand Up @@ -72,6 +75,10 @@ func (c *Config) ReadConfigFile(cfg_path string) error {
c.TLSCrt = c.NodeName + ".crt"
}

if c.NodeSSHKey == "" {
c.NodeSSHKey = c.NodeName + "_id_rsa"
}

_, err := time.ParseDuration(c.DefaultResourceLifetime)
if c.DefaultResourceLifetime != "" && err != nil {
return fmt.Errorf("Fish: Default Resource Lifetime parse error: %v", err)
Expand All @@ -83,7 +90,8 @@ func (c *Config) ReadConfigFile(cfg_path string) error {
func (c *Config) initDefaults() {
c.Directory = "fish_data"
c.APIAddress = "0.0.0.0:8001"
c.ProxyAddress = "0.0.0.0:1080"
c.ProxySocksAddress = "0.0.0.0:1080"
c.ProxySshAddress = "0.0.0.0:2022"
c.NodeAddress = "127.0.0.1:8001"
c.TLSKey = "" // Will be set after read config file from NodeName
c.TLSCrt = "" // ...
Expand Down
2 changes: 2 additions & 0 deletions lib/fish/fish.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ func (f *Fish) Init() error {
&types.ApplicationState{},
&types.ApplicationTask{},
&types.Resource{},
&types.ResourceAccess{},
&types.Vote{},
&types.Location{},
&types.ServiceMapping{},
Expand Down Expand Up @@ -616,6 +617,7 @@ func (f *Fish) executeApplication(vote types.Vote) error {
res.IpAddr = drv_res.IpAddr
res.LabelUID = label.UID
res.DefinitionIndex = vote.Available
res.Authentication = drv_res.Authentication
err := f.ResourceCreate(res)
if err != nil {
log.Error("Fish: Unable to store Resource for Application:", app.UID, err)
Expand Down
6 changes: 6 additions & 0 deletions lib/fish/resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@ func (f *Fish) ResourceCreate(r *types.Resource) error {
}

func (f *Fish) ResourceDelete(uid types.ResourceUID) error {
// First delete any references to this resource.
err := f.ResourceAccessDeleteByResource(uid)
if err != nil {
log.Errorf("Unable to delete ResourceAccess associated with Resource UID=%v: %v", uid, err)
}
// Now purge the resource.
return f.db.Delete(&types.Resource{}, uid).Error
}

Expand Down
Loading

0 comments on commit fb07ac5

Please sign in to comment.