Commit 86227e47 authored by Pierre Smeyers's avatar Pierre Smeyers
Browse files

Merge branch 'feat/crud-operations' into 'master'

Support CRUD operations on secrets

See merge request to-be-continuous/tools/vault-secrets-provider!16
parents 284112c3 806adc06
Loading
Loading
Loading
Loading
+54 −8
Original line number Diff line number Diff line
@@ -85,17 +85,63 @@ test-on-local:
    - curl -sSf "$VAULT_BASE_URL/sys/health"
    # create a secret in Vault
    - |
      curl --silent --header "X-Vault-Token: ${VAULT_DEV_ROOT_TOKEN_ID}" --request PUT --data '{"options": {"cas": 0}, "data": {"foo": "bar", "zip": "zap"}}' "${VAULT_BASE_URL}/secret/data/my-secret"
    # now check we can retrieve the secret through Vault Secrets Provider
      curl --silent --header "X-Vault-Token: ${VAULT_DEV_ROOT_TOKEN_ID}" --request PUT --data '{"data": {"foo": "bar", "zip": "zap"}}' "${VAULT_BASE_URL}/secret/data/my-secret"
    # test: get existing secret shall succeed
    - |
      if foo_secret=$(curl -sSf "http://vault-secrets-provider/api/secrets/my-secret?field=foo")
      then
        echo "secret retrieved - $foo_secret"
      else
        echo "FAILED retrieving secret"
        curl --silent "http://vault-secrets-provider/api/secrets/my-secret?field=foo"
      resp_status=$(curl -s -o "resp.txt" -w "%{http_code}" "http://vault-secrets-provider/api/secrets/my-secret?field=foo")
      if [[ "$resp_status" != "200" ]]; then
        echo "FAILED get existing secret ($resp_status)"
        cat resp.txt
        exit 1
      fi
    # test: get secret with non existing path shall fail with code 404
    - |
      resp_status=$(curl -s -o "resp.txt" -w "%{http_code}" "http://vault-secrets-provider/api/secrets/no/such/path?field=foo")
      if [[ "$resp_status" != "404" ]]; then
        echo "FAILED get secret with non existing path ($resp_status)"
        cat resp.txt
        exit 1
      fi
    # test: get secret with non existing field shall fail with code 400
    - |
      resp_status=$(curl -s -o "resp.txt" -w "%{http_code}" "http://vault-secrets-provider/api/secrets/my-secret?field=baz")
      if [[ "$resp_status" != "400" ]]; then
        echo "FAILED get secret with non existing field ($resp_status)"
        cat resp.txt
        exit 1
      fi
    # test: create secret shall succeed
    - |
      resp_status=$(curl -s -o "resp.txt" -w "%{http_code}" -X PUT --data '{"key":"value"}' "http://vault-secrets-provider/api/secrets/new-secret")
      if [[ "$resp_status" != "200" ]]; then
        echo "FAILED create secret ($resp_status)"
        cat resp.txt
        exit 1
      fi
    # test: get created secret shall succeed
    - |
      resp_status=$(curl -s -o "resp.txt" -w "%{http_code}" "http://vault-secrets-provider/api/secrets/new-secret?field=key")
      if [[ "$resp_status" != "200" ]]; then
        echo "FAILED get created secret ($resp_status)"
        cat resp.txt
        exit 1
      fi
    # test: delete secret shall succeed
    - |
      resp_status=$(curl -s -o "resp.txt" -w "%{http_code}" -X DELETE "http://vault-secrets-provider/api/secrets/new-secret")
      if [[ "$resp_status" != "204" ]]; then
        echo "FAILED delete secret ($resp_status)"
        cat resp.txt
        exit 1
      fi
    # # test: get deleted secret shall fail with code 404
    # - |
    #   resp_status=$(curl -s -o "resp.txt" -w "%{http_code}" "http://vault-secrets-provider/api/secrets/new-secret?field=key")
    #   if [[ "$resp_status" != "404" ]]; then
    #     echo "FAILED get secret with non existing path ($resp_status)"
    #     cat resp.txt
    #     exit 1
    #   fi
  only:
    refs:
      - branches
+66 −4
Original line number Diff line number Diff line
@@ -28,26 +28,28 @@ This way:

### API overview

The service exposes one single API to read a secret from the Vault server.
The service exposes one single API to read, create/update or delete a secret from the Vault server.

:warning: the service is smart enough to auto-detect whether the Vault server is configured to use KV Secrets Engine
version 1 (unversioned mode) or version 2 (versioned mode). Thus you don't need to worry about the `data` part
in the resource path or in the response object structure.

#### Endpoint
#### GET secret endpoint

Read a secret from the Vault server, seamlessly using the right API depending on the detected Key-Value engine version ([version 1](https://www.vaultproject.io/api-docs/secret/kv/kv-v1#read-secret) or [version 2](https://www.vaultproject.io/api-docs/secret/kv/kv-v2#read-secret-version)).

```curl
GET /api​/secrets​/{secret_path}
```

#### Parameters
##### Parameters

| Name                             | description                            |
| -------------------------------- | -------------------------------------- |
| `secret_path` (_path parameter_) | this is your secret location in the Vault server |
| `field` (_query parameter_)      | parameter to access a single basic field from the secret JSON payload |

#### Example
##### Example

Let's suppose your have a secret stored under `/b7ecb6ebabc231/my-backend/prod` location in Vault with JSON payload:

@@ -67,6 +69,66 @@ Then you may retrieve:
* the token calling `GET http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/my-backend/prod?field=token`
* the MySql password calling `GET http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/my-backend/prod?field=mysql.password`

#### PUT secret endpoint

Create/Update a secret into the Vault server, seamlessly using the right API depending on the detected Key-Value engine version ([version 1](https://www.vaultproject.io/api-docs/secret/kv/kv-v1#create-update-secret) or [version 2](https://www.vaultproject.io/api-docs/secret/kv/kv-v2#create-update-secret)).

```curl
PUT /api​/secrets​/{secret_path}
 -d '{"token": "@n0ther_ŧ0k3N", "secret": "n€w_$€cr€T"}' 
 -H "Content-Type: application/json"
```

:warning: depending on the Key-Value engine version the Vault server is using, this might either overwrite the secret (v1) or create a new version (v2).
When using version 2, you don't need to embed your secret payload into a `data` field - this is done automatically for you.

##### Parameters

| Name                             | description                            |
| -------------------------------- | -------------------------------------- |
| `secret_path` (_path parameter_) | this is your secret location in the Vault server |

##### Example

Create/update a secret under `/b7ecb6ebabc231/my-backend/review` location in Vault with JSON payload:

```curl
curl -X PUT http://localhost:8080/api/secret/b7ecb6ebabc231/my-backend/review \
 -d '{"token": "@n0ther_ŧ0k3N", "secret": "n€w_$€cr€T"}' 
 -H "Content-Type: application/json"
```

```json
{
  "created_time":"2021-11-10T10:40:49.286084835Z",
  "deletion_time":"",
  "destroyed":false,
  "version":1
}
```

#### DELETE secret endpoint

Delete a secret from the Vault server, seamlessly using the right API depending on the detected Key-Value engine version ([version 1](https://www.vaultproject.io/api-docs/secret/kv/kv-v1#delete-secret) or [version 2](https://www.vaultproject.io/api-docs/secret/kv/kv-v2#delete-latest-version-of-secret)).

```curl
DELETE /api​/secrets​/{secret_path}
```

##### Parameters

| Name                             | description                            |
| -------------------------------- | -------------------------------------- |
| `secret_path` (_path parameter_) | this is your secret location in the Vault server |

##### Example

Delete a secret stored under `/b7ecb6ebabc231/my-backend/review` location in Vault with JSON payload:

```curl
curl -X DELETE http://localhost:8080/api/secret/b7ecb6ebabc231/my-backend/review
```

### Required image parameters

The tool requires the following environment variables to be set (as GitLab CI secret variables):
+318 −29
Original line number Diff line number Diff line
@@ -20,7 +20,10 @@ package internal
import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
@@ -67,23 +70,36 @@ var (
	vaultJwtRole             = os.Getenv("VAULT_JWT_ROLE")
	vaultRoleId              = os.Getenv("VAULT_ROLE_ID")
	vaultSecretId            = os.Getenv("VAULT_SECRET_ID")
	clientToken              = os.Getenv("VAULT_TOKEN")
	expirationTimeSec        = time.Now().Unix() + 86400 // 24 hours from start time
	kvEngineVersion          = 0
)

func DumpVaultCfg() {
	log.Printf("Vault server base url: '%s'\n", vaultBaseUrl)
	if len(clientToken) > 0 {
		log.Printf("Vault Auth method: Token (provided)\n")
	} else if len(vaultRoleId) > 0 && len(vaultSecretId) > 0 {
		log.Printf("Vault Auth method: AppRole (path: '%s')\n", vaultBaseAuthApprolePath)
	} else if len(vaultJwtToken) > 0 {
		log.Printf("Vault Auth method: JWT (path: '%s')\n", vaultBaseAuthJwtPath)
	} else {
		log.Printf("WARN: no Vault Auth method configured\n")
	}
	log.Printf("Vault namespace: '%s'\n", vaultNamespace)
	log.Printf("Vault base auth/JWT path: '%s'\n", vaultBaseAuthJwtPath)
	log.Printf("Vault base auth/AppRole path: '%s'\n", vaultBaseAuthApprolePath)
	log.Printf("Vault base secrets/KV path: '%s'\n", vaultBaseSecretsKvPath)
}

var clientToken = os.Getenv("VAULT_TOKEN")        // an unlimited token might passed (dev only)
var expirationTimeSec = time.Now().Unix() + 86400 // 24 hours from start time

/**
 * Determines whether the token is valid or not
 */
func hasValidToken() bool {
	return len(clientToken) > 0 && time.Now().Unix() < (expirationTimeSec-1)
}

/**
 * This function returns a valid authentication token, caching the value (thus the login request to the Vault server is executed only once)
 */
func getToken() (string, error) {
	if !hasValidToken() {
		if auth, err := doLogin(); err != nil {
@@ -101,23 +117,10 @@ func getToken() (string, error) {
	return clientToken, nil
}

var kvEngineVersion = 0

func getKvEngineVersion(secretPath string) (int, error) {
	if kvEngineVersion == 0 {
		// need to determine actual KV engine version on server
		if version, err := doGetKvEngineVersion(secretPath); err != nil {
			return 0, err
		} else {
			kvEngineVersion = version
		}
	}
	return kvEngineVersion, nil
}

/**
 * Performs the login request
 * either
 * This function requests the Vault server endpoint to perform the login (depending on the used Auth method)
 * - AppRole Auth method: https://www.vaultproject.io/api-docs/auth/approle#login-with-approle
 * - JWT/OIDC Auth method: https://www.vaultproject.io/api-docs/auth/jwt#jwt-login
 */
func doLogin() (*Authentication, error) {
	var loginObj interface{}
@@ -201,6 +204,25 @@ func doLogin() (*Authentication, error) {
	}
}

/**
 * This function returns the KV engine version, caching the value (thus the request to the Vault endpoint is executed only once)
 */
func getKvEngineVersion(secretPath string) (int, error) {
	if kvEngineVersion == 0 {
		// need to determine actual KV engine version on server
		if version, err := doGetKvEngineVersion(secretPath); err != nil {
			return 0, err
		} else {
			kvEngineVersion = version
		}
	}
	return kvEngineVersion, nil
}

/**
 * This function requests the Vault server endpoint to read the KV engine version
 * https://www.vaultproject.io/api-docs/system/internal-ui-mounts
 */
func doGetKvEngineVersion(secretPath string) (int, error) {
	url := fmt.Sprintf("%s/sys/internal/ui/mounts%s/%s", vaultBaseUrl, vaultBaseSecretsKvPath, secretPath)

@@ -270,6 +292,12 @@ func doGetKvEngineVersion(secretPath string) (int, error) {
	}
}

/**
 * This function requests the Vault server endpoint to read a (latest version of a) secret.
 * It manages seamlessly the KV engine version
 * - version 1: https://www.vaultproject.io/api-docs/secret/kv/kv-v1#read-secret
 * - version 2: https://www.vaultproject.io/api-docs/secret/kv/kv-v2#read-secret-version
 */
func doGetSecret(secretPath string) (map[string]interface{}, error) {
	// 1: determine KV engine version
	if version, err := getKvEngineVersion(secretPath); err != nil {
@@ -346,6 +374,10 @@ func doGetSecret(secretPath string) (map[string]interface{}, error) {
	}
}

/**
 * This function returns the given secret field, caching the JSON value (thus the request to read the JSON secret is executed only once,
 * even if subsequent calls are made to read other fields of the same secret)
 */
func getSecret(secretPath string, keyPath string) (string, error) {
	log.Printf("Get secret '%s' (%s)\n", secretPath, keyPath)

@@ -384,6 +416,155 @@ func getSecret(secretPath string, keyPath string) (string, error) {
	}
}

/**
 * This function requests the Vault server endpoint to create/update a (new version of a) secret.
 * It manages seamlessly the KV engine version
 * - version 1: https://www.vaultproject.io/api-docs/secret/kv/kv-v1#create-update-secret
 * - version 2: https://www.vaultproject.io/api-docs/secret/kv/kv-v2#create-update-secret
 */
func doUpdateSecret(secretPath string, body string) (map[string]interface{}, error) {
	// 1: determine KV engine version
	if version, err := getKvEngineVersion(secretPath); err != nil {
		return nil, StatusError{
			Code:    http.StatusInternalServerError,
			Message: "Failed retrieving KV engine version",
			Err:     err,
		}
	} else {
		if version == 2 {
			// transform the request path
			secretPath = "data/" + secretPath
			// and also the JSON structure
			body = "{\"data\": " + body + "}"
		}
	}
	url := fmt.Sprintf("%s%s/%s", vaultBaseUrl, vaultBaseSecretsKvPath, secretPath)
	if request, err := http.NewRequest(http.MethodPost, url, bytes.NewBufferString(body)); err != nil {
		// create request error: propagate
		return nil, StatusError{
			Code:    http.StatusInternalServerError,
			Message: "Failed creating request",
			Err:     err,
		}
	} else if token, err := getToken(); err != nil {
		// login error: propagate
		return nil, StatusError{
			Message: "Login failed",
			Err:     err,
		}
	} else {
		request.Header.Set("Content-Type", "application/json")
		request.Header.Set(ClientTokenHeader, token)
		if len(vaultNamespace) > 0 {
			request.Header.Set(NamespaceHeader, vaultNamespace)
		}

		log.Printf("... create secret '%s' (POST %s)\n", secretPath, url)
		client := &http.Client{}
		if resp, err := client.Do(request); err != nil {
			// request error: propagate
			return nil, StatusError{
				Code:    http.StatusInternalServerError,
				Message: "Failed sending request",
				Err:     err,
			}
		} else if resp.StatusCode != http.StatusOK {
			// http error: decode error body and throw error
			defer resp.Body.Close()
			var errResp ErrorResponse
			// nolint
			json.NewDecoder(resp.Body).Decode(&errResp)

			return nil, StatusError{
				Code:    resp.StatusCode,
				Message: fmt.Sprintf("Vault server error on POST %s", url),
				Err:     errResp,
			}
		} else {
			// OK: decode response
			defer resp.Body.Close()
			var secret UpdateSecretResponse
			if err := json.NewDecoder(resp.Body).Decode(&secret); err != nil {
				// JSON unmarshall error: propagate
				return nil, StatusError{
					Code:    http.StatusInternalServerError,
					Message: "Failed unmarshalling response body",
					Err:     err,
				}
			}
			// log.Printf("... secret %s retrieved: %s\n", secretPath, secret.Data)
			return secret.Data, nil
		}
	}
}

/**
 * This function requests the Vault server endpoint to delete the (latest version of the) secret.
 * It manages seamlessly the KV engine version
 * - version 1: https://www.vaultproject.io/api-docs/secret/kv/kv-v1#delete-secret
 * - version 2: https://www.vaultproject.io/api-docs/secret/kv/kv-v2#delete-latest-version-of-secret
 */
func doDeleteSecret(secretPath string) error {
	// 1: determine KV engine version
	if version, err := getKvEngineVersion(secretPath); err != nil {
		return StatusError{
			Code:    http.StatusInternalServerError,
			Message: "Failed retrieving KV engine version",
			Err:     err,
		}
	} else {
		if version == 2 {
			secretPath = "data/" + secretPath
		}
	}
	url := fmt.Sprintf("%s%s/%s", vaultBaseUrl, vaultBaseSecretsKvPath, secretPath)
	if request, err := http.NewRequest(http.MethodDelete, url, nil); err != nil {
		// create request error: propagate
		return StatusError{
			Code:    http.StatusInternalServerError,
			Message: "Failed creating request",
			Err:     err,
		}
	} else if token, err := getToken(); err != nil {
		// login error: propagate
		return StatusError{
			Message: "Login failed",
			Err:     err,
		}
	} else {
		request.Header.Set("Content-Type", "application/json")
		request.Header.Set(ClientTokenHeader, token)
		if len(vaultNamespace) > 0 {
			request.Header.Set(NamespaceHeader, vaultNamespace)
		}

		log.Printf("... delete secret '%s' (DELETE %s)\n", secretPath, url)
		client := &http.Client{}
		if resp, err := client.Do(request); err != nil {
			// request error: propagate
			return StatusError{
				Code:    http.StatusInternalServerError,
				Message: "Failed sending request",
				Err:     err,
			}
		} else if resp.StatusCode != http.StatusNoContent {
			// http error: decode error body and throw error
			defer resp.Body.Close()
			var errResp ErrorResponse
			// nolint
			json.NewDecoder(resp.Body).Decode(&errResp)

			return StatusError{
				Code:    resp.StatusCode,
				Message: fmt.Sprintf("Vault server error on DELETE %s", url),
				Err:     errResp,
			}
		} else {
			return nil
		}
	}
}

/**
 * Retrieves a sub object within a map
 * @arg obj: a map object
@@ -421,14 +602,81 @@ func getObj(obj map[string]interface{}, keyPath string) (string, error) {
}

/**
 * get secret API endpoint /api/secrets
 * Check json format into request body
 */
func checkBody(request *http.Request) (int, []byte, error) {
	bodyStr, _ := ioutil.ReadAll(request.Body)
	if len(bodyStr) <= 0 {
		return http.StatusBadRequest, nil, fmt.Errorf("request body must not be empty")
	}
	err := json.Unmarshal(bodyStr, &struct{}{})
	if err != nil {
		var syntaxError *json.SyntaxError
		switch {
		// Catch any syntax errors in the JSON and send an error message
		// which interpolates the location of the problem to make it
		// easier for the client to fix.
		case errors.As(err, &syntaxError):
			// http.Error(writer, msg, http.StatusBadRequest)
			return http.StatusBadRequest, nil, fmt.Errorf("request body contains badly-formed json (at position %d)", syntaxError.Offset)

		// In some circumstances Decode() may also return an
		// io.ErrUnexpectedEOF error for syntax errors in the JSON. There
		// is an open issue regarding this at
		// https://github.com/golang/go/issues/25956.
		case errors.Is(err, io.ErrUnexpectedEOF):
			// http.Error(writer, msg, http.StatusBadRequest)
			return http.StatusBadRequest, nil, fmt.Errorf("request body contains badly-formed json")

		// An io.EOF error is returned by Decode() if the request body is
		// empty.
		// case errors.Is(err, io.EOF):
		// 	// http.Error(writer, msg, http.StatusBadRequest)
		// 	return http.StatusBadRequest, nil, fmt.Errorf("request body must not be empty")

		// Catch the error caused by the request body being too large. Again
		// there is an open issue regarding turning this into a sentinel
		// error at https://github.com/golang/go/issues/30715.
		case err.Error() == "http: request body too large":
			// http.Error(writer, msg, http.StatusRequestEntityTooLarge)
			return http.StatusRequestEntityTooLarge, nil, fmt.Errorf("request body must not be larger than 1mb")

		// Otherwise default to logging the error and sending a 500 Internal
		// Server Error response.
		default:
			return http.StatusInternalServerError, nil, fmt.Errorf(err.Error())
			// log.Println(err.Error())
			// http.Error(writer, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		}
	}
	return 200, bodyStr, nil
}

/**
 * This is the /api/secrets API endpoint
 * Routes the incoming request depending on the Http method:
 * - GET /api/secrets/{path}
 * - PUT/POST /api/secrets/{path}
 * - DELETE /api/secrets/{path}
 */
func SecretsEndpoint(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		GetSecret(w, r)
	case "POST", "PUT":
		UpdateSecret(w, r)
	case "DELETE":
		DeleteSecret(w, r)
	default:
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
	}
}

/**
 * GET /api/secrets endpoint function
 */
func GetSecret(writer http.ResponseWriter, request *http.Request) {
	writer.Header().Set("Content-Type", "text/plain")

	if strings.ToUpper(request.Method) != "GET" {
		http.Error(writer, "Method not allowed", http.StatusMethodNotAllowed)
	} else {
	secretPath := request.URL.Path[13:]
	keyPath := request.URL.Query().Get("field")

@@ -442,4 +690,45 @@ func GetSecret(writer http.ResponseWriter, request *http.Request) {
		}
	}
}

/**
 * PUT /api/secrets endpoint function
 */
func UpdateSecret(writer http.ResponseWriter, request *http.Request) {
	writer.Header().Set("Content-Type", "application/json")
	secretPath := request.URL.Path[13:]

	if status, body, err := checkBody(request); err != nil {
		log.Printf("%d %s", status, err)
		http.Error(writer, err.Error(), status)
	} else {
		if createResp, err := doUpdateSecret(secretPath, string(body)); err != nil {
			http.Error(writer, err.Error(), err.(StatusError).Status())
		} else {
			delete(path2secret, secretPath)
			createJson, err := json.Marshal(createResp)
			if err != nil {
				http.Error(writer, err.Error(), http.StatusInternalServerError)
				return
			}
			writer.WriteHeader(http.StatusOK)
			if _, err = writer.Write([]byte(createJson)); err != nil {
				log.Println("Error while sending response")
			}
		}
	}
}

/**
 * DELETE /api/secrets endpoint function
 */
func DeleteSecret(writer http.ResponseWriter, request *http.Request) {
	secretPath := request.URL.Path[13:]

	if err := doDeleteSecret(secretPath); err != nil {
		http.Error(writer, err.Error(), err.(StatusError).Status())
	} else {
		delete(path2secret, secretPath)
		writer.WriteHeader(http.StatusNoContent)
	}
}
+10 −0
Original line number Diff line number Diff line
@@ -113,3 +113,13 @@ type GetSecretResponse struct {
	Renewable     bool                   `json:"renewable"`
	Data          map[string]interface{} `json:"data"`
}

type UpdateSecretResponse struct {
	WrapInfo      map[string]string      `json:"wrap_info"`
	LeaseDuration int64                  `json:"lease_duration"`
	LeaseId       string                 `json:"lease_id"`
	RequestId     string                 `json:"request_id"`
	Warnings      []string               `json:"warnings"`
	Renewable     bool                   `json:"renewable"`
	Data          map[string]interface{} `json:"data"`
}
+21 −0
Original line number Diff line number Diff line
@@ -101,6 +101,27 @@ func Test_get_secret_response_unmarshalling(t *testing.T) {
	}
}

func Test_update_secret_response_unmarshalling(t *testing.T) {
	response := `{
		"auth": null,
		"data": {
		  "created_time": "2021-11-10T10:11:12.400489065Z",
		  "deletion_time": "",
			"destroyed": false,
			"version": 1
		},
		"lease_duration": 0,
		"lease_id": "",
		"renewable": false
	  }`
	var updateSecretResp UpdateSecretResponse
	if err := json.Unmarshal([]byte(response), &updateSecretResp); err != nil {
		t.Fatalf("Error while unmarshalling message: %v", err)
	} else if updateSecretResp.Data["created_time"] != "2021-11-10T10:11:12.400489065Z" {
		t.Fatalf("Decoding error\nExpected:\n%s\nGot:\n%s", "2021-11-10T10:11:12.400489065Z", updateSecretResp.Data["created_time"])
	}
}

func Test_get_error_unmarshalling_simple(t *testing.T) {
	response := `{
		"errors": ["invalid role ID"]
Loading