Merge branch 'feat/patch-secret' into 'master' (baba4bf4) · Commits · to be continuous... / tools / vault-secrets-provider

.gitlab-ci.yml

+30 −0

Original line number	Diff line number	Diff line
		@@ -129,6 +129,36 @@ test-on-local:
		cat resp.txt
		exit 1
		fi
		# test: patch secret shall succeed
		- \|
		resp_status=$(curl -s -o "resp.txt" -w "%{http_code}" -X PATCH --data '{"key":"new-value","to-patch":"not-patched","other":"value"}' "http://vault-secrets-provider/api/secrets/new-secret")
		if [[ "$resp_status" != "200" ]]; then
		echo "FAILED patch secret ($resp_status)"
		cat resp.txt
		exit 1
		fi
		# test: patch secret field shall succeed
		- \|
		resp_status=$(curl -s -o "resp.txt" -w "%{http_code}" -X PATCH --data 'patched' "http://vault-secrets-provider/api/secrets/new-secret?field=to-patch")
		if [[ "$resp_status" != "200" ]]; then
		echo "FAILED patch secret field ($resp_status)"
		cat resp.txt
		exit 1
		fi
		# test: get patched secret shall succeed
		- \|
		resp_status=$(curl -s -o "resp.txt" -w "%{http_code}" "http://vault-secrets-provider/api/secrets/new-secret?field=to-patch")
		if [[ "$resp_status" != "200" ]]; then
		echo "FAILED get patched secret ($resp_status)"
		cat resp.txt
		exit 1
		fi
		if [[ "$(cat resp.txt)" != "patched" ]]; then
		echo "FAILED get patched secret content"
		echo "Expected: patched"
		echo "Got: $(cat resp.txt)"
		exit 1
		fi
		# test: delete secret shall succeed
		- \|
		resp_status=$(curl -s -o "resp.txt" -w "%{http_code}" -X DELETE "http://vault-secrets-provider/api/secrets/new-secret")

README.md

+38 −0

Original line number	Diff line number	Diff line
		@@ -111,6 +111,44 @@ curl -X PUT http://localhost:8080/api/secret/b7ecb6ebabc231/my-backend/review \
		}
		```

		#### PATCH secret endpoint

		Partially update a secret in the Vault server, seamlessly depending on the detected Key-Value engine version. This uses native [version 2 `PATCH`](https://www.vaultproject.io/api-docs/secret/kv/kv-v2#patch-secret) API and it implemented using Read, Merge then Write for version 1.

		##### Single Field Update

		Patch a single field of a secret path generating a [JSON Patch](https://datatracker.ietf.org/doc/html/rfc6902) for you.

		```curl
		PATCH /api/secrets/{secret_path}?field={field_name}
		-d '{secret_value}'
		-H "Content-Type: text/plain"
		```

		###### Parameters

		\| Name \| Description \|
		\| -------------------------------- \| -------------------------------------- \|
		\| `secret_path` (_path parameter_) \| this is your secret location in the Vault server \|
		\| `field_name` (_query parameter_) \| the name of the field to update in the secret \|
		\| `secret_value` (_request body_) \| the value to set for the field in the secret \|

		##### JSON Patch

		Patch the whole secret path using a [JSON Merge Patch](https://datatracker.ietf.org/doc/html/rfc7386) request body.

		```curl
		PUT /api/secrets/{secret_path}
		-d '{"{field_name}": "{secret_value}", "other_key": "other_secret"}'
		-H "Content-Type: application/merge-patch+json"
		```

		###### Parameters

		\| Name \| Description \|
		\| -------------------------------- \| -------------------------------------- \|
		\| `secret_path` (_path parameter_) \| this is your secret location in the Vault server \|

		#### DELETE secret endpoint

		Delete a secret from the Vault server, seamlessly using the right API depending on the detected Key-Value engine version ([version 1](https://www.vaultproject.io/api-docs/secret/kv/kv-v1#delete-secret) or [version 2](https://www.vaultproject.io/api-docs/secret/kv/kv-v2#delete-latest-version-of-secret)).

cmd/vault_service/internal/vault_client.go

+140 −136

Original line number	Diff line number	Diff line
		@@ -293,7 +293,7 @@ func doGetKvEngineVersion(secretPath string) (int, error) {
		} else {
		// OK: decode response
		defer resp.Body.Close() // nolint:errcheck
		var getUiMountsResp GetSecretResponse
		var getUiMountsResp DataResponse
		if err := json.NewDecoder(resp.Body).Decode(&getUiMountsResp); err != nil {
		// JSON unmarshall error: propagate
		return 0, StatusError{
		@@ -315,14 +315,7 @@ func doGetKvEngineVersion(secretPath string) (int, error) {
		}
		}

		/**
		* This function requests the Vault server endpoint to read a (latest version of a) secret.
		* It manages seamlessly the KV engine version
		* - version 1: https://www.vaultproject.io/api-docs/secret/kv/kv-v1#read-secret
		* - version 2: https://www.vaultproject.io/api-docs/secret/kv/kv-v2#read-secret-version
		*/
		func doGetSecret(secretPath string) (map[string]interface{}, error) {
		// 1: determine KV engine version
		func doSecretRequest(method string, secretPath string, payload string) (map[string]interface{}, error) {
		if version, err := getKvEngineVersion(secretPath); err != nil {
		return nil, StatusError{
		Code: http.StatusInternalServerError,
		@@ -331,14 +324,22 @@ func doGetSecret(secretPath string) (map[string]interface{}, error) {
		}
		} else {
		if version == 2 {
		// append "data/"
		// transform the request path
		secretPath = "data/" + secretPath
		// and also the JSON structure
		if len(payload) > 0 {
		payload = "{\"data\": " + payload + "}"
		}
		}
		}

		url := fmt.Sprintf("%s%s/%s", vaultBaseUrl, vaultBaseSecretsKvPath, secretPath)
		var body io.Reader = nil
		if len(payload) > 0 {
		body = bytes.NewBufferString(payload)
		}

		if request, err := http.NewRequest(http.MethodGet, url, nil); err != nil {
		if request, err := http.NewRequest(method, url, body); err != nil {
		// create request error: propagate
		return nil, StatusError{
		Code: http.StatusInternalServerError,
		@@ -352,13 +353,18 @@ func doGetSecret(secretPath string) (map[string]interface{}, error) {
		Err: err,
		}
		} else {
		request.Header.Set("Content-Type", "application/json")
		contentType := "application/json"
		if method == http.MethodPatch {
		contentType = "application/merge-patch+json"
		}
		request.Header.Set("Content-Type", contentType)
		request.Header.Set(ClientTokenHeader, token)
		if len(vaultNamespace) > 0 {
		request.Header.Set(NamespaceHeader, vaultNamespace)
		}

		log.Printf("... retrieve secret '%s' (GET %s)\n", secretPath, url)
		log.Printf("... requesting secret '%s' (%s %s)\n", secretPath, method, url)
		delete(path2secret, secretPath) // remove from cache
		client := &http.Client{}
		if resp, err := client.Do(request); err != nil {
		// request error: propagate
		@@ -367,6 +373,8 @@ func doGetSecret(secretPath string) (map[string]interface{}, error) {
		Message: "Failed sending request",
		Err: err,
		}
		} else if resp.StatusCode == http.StatusNoContent {
		return nil, nil
		} else if resp.StatusCode != http.StatusOK {
		// http error: decode error body and throw error
		defer resp.Body.Close() // nolint:errcheck
		@@ -382,7 +390,7 @@ func doGetSecret(secretPath string) (map[string]interface{}, error) {
		} else {
		// OK: decode response
		defer resp.Body.Close() // nolint:errcheck
		var secret GetSecretResponse
		var secret DataResponse
		if err := json.NewDecoder(resp.Body).Decode(&secret); err != nil {
		// JSON unmarshall error: propagate
		return nil, StatusError{
		@@ -397,6 +405,16 @@ func doGetSecret(secretPath string) (map[string]interface{}, error) {
		}
		}

		/**
		* This function requests the Vault server endpoint to read a (latest version of a) secret.
		* It manages seamlessly the KV engine version
		* - version 1: https://www.vaultproject.io/api-docs/secret/kv/kv-v1#read-secret
		* - version 2: https://www.vaultproject.io/api-docs/secret/kv/kv-v2#read-secret-version
		*/
		func doGetSecret(secretPath string) (map[string]interface{}, error) {
		return doSecretRequest(http.MethodGet, secretPath, "")
		}

		/**
		* This function returns the given secret field, caching the JSON value (thus the request to read the JSON secret is executed only once,
		* even if subsequent calls are made to read other fields of the same secret)
		@@ -446,78 +464,62 @@ func getSecret(secretPath string, keyPath string) (string, error) {
		* - version 2: https://www.vaultproject.io/api-docs/secret/kv/kv-v2#create-update-secret
		*/
		func doUpdateSecret(secretPath string, body string) (map[string]interface{}, error) {
		// 1: determine KV engine version
		if version, err := getKvEngineVersion(secretPath); err != nil {
		return doSecretRequest(http.MethodPost, secretPath, body)
		}

		/**
		* This function requests the Vault server endpoint to do a partial update of a secret.
		* It manages seamlessly the KV engine version
		* - version 1: https://developer.hashicorp.com/vault/api-docs/secret/kv/kv-v1#read-secret then https://www.vaultproject.io/api-docs/secret/kv/kv-v1#create-update-secret
		* - version 2: https://developer.hashicorp.com/vault/api-docs/secret/kv/kv-v2#patch-secret
		*/
		func doPatchSecret(secretPath string, body string) (map[string]interface{}, error) {
		version, err := getKvEngineVersion(secretPath)
		if err != nil {
		return nil, StatusError{
		Code: http.StatusInternalServerError,
		Message: "Failed retrieving KV engine version",
		Err: err,
		}
		} else {
		}

		if version == 2 {
		// transform the request path
		secretPath = "data/" + secretPath
		// and also the JSON structure
		body = "{\"data\": " + body + "}"
		// version 2: just PATCH request
		return doSecretRequest(http.MethodPatch, secretPath, body)
		}

		// version 1: first read the secret, then update it
		secretMap, err := doGetSecret(secretPath)
		if err != nil {
		return nil, err
		}
		url := fmt.Sprintf("%s%s/%s", vaultBaseUrl, vaultBaseSecretsKvPath, secretPath)
		if request, err := http.NewRequest(http.MethodPost, url, bytes.NewBufferString(body)); err != nil {
		// create request error: propagate

		// then update the secret with the new values
		var secretPatch map[string]interface{}
		if err := json.Unmarshal([]byte(body), &secretPatch); err != nil {
		return nil, StatusError{
		Code: http.StatusInternalServerError,
		Message: "Failed creating request",
		Code: http.StatusBadRequest,
		Message: "Invalid JSON body",
		Err: err,
		}
		} else if token, err := getToken(); err != nil {
		// login error: propagate
		return nil, StatusError{
		Message: "Login failed",
		Err: err,
		}
		// NOTE: this is far from a full implementation of merge patch json (rfc7386)
		for key, value := range secretPatch {
		if value == nil {
		delete(secretMap, key)
		} else {
		request.Header.Set("Content-Type", "application/json")
		request.Header.Set(ClientTokenHeader, token)
		if len(vaultNamespace) > 0 {
		request.Header.Set(NamespaceHeader, vaultNamespace)
		secretMap[key] = value
		}

		log.Printf("... create secret '%s' (POST %s)\n", secretPath, url)
		client := &http.Client{}
		if resp, err := client.Do(request); err != nil {
		// request error: propagate
		return nil, StatusError{
		Code: http.StatusInternalServerError,
		Message: "Failed sending request",
		Err: err,
		}
		} else if resp.StatusCode != http.StatusOK {
		// http error: decode error body and throw error
		defer resp.Body.Close() // nolint:errcheck
		var errResp ErrorResponse
		// nolint
		json.NewDecoder(resp.Body).Decode(&errResp)

		return nil, StatusError{
		Code: resp.StatusCode,
		Message: fmt.Sprintf("Vault server error on POST %s", url),
		Err: errResp,
		}
		} else {
		// OK: decode response
		defer resp.Body.Close() // nolint:errcheck
		var secret UpdateSecretResponse
		if err := json.NewDecoder(resp.Body).Decode(&secret); err != nil {
		// JSON unmarshall error: propagate
		if newBody, err := json.Marshal(secretMap); err != nil {
		return nil, StatusError{
		Code: http.StatusInternalServerError,
		Message: "Failed unmarshalling response body",
		Message: "Failed marshalling updated secret",
		Err: err,
		}
		}
		// log.Printf("... secret %s retrieved: %s\n", secretPath, secret.Data)
		return secret.Data, nil
		}
		} else {
		// then update the secret
		return doUpdateSecret(secretPath, string(newBody))
		}
		}

		@@ -528,64 +530,8 @@ func doUpdateSecret(secretPath string, body string) (map[string]interface{}, err
		* - version 2: https://www.vaultproject.io/api-docs/secret/kv/kv-v2#delete-latest-version-of-secret
		*/
		func doDeleteSecret(secretPath string) error {
		// 1: determine KV engine version
		if version, err := getKvEngineVersion(secretPath); err != nil {
		return StatusError{
		Code: http.StatusInternalServerError,
		Message: "Failed retrieving KV engine version",
		Err: err,
		}
		} else {
		if version == 2 {
		secretPath = "data/" + secretPath
		}
		}
		url := fmt.Sprintf("%s%s/%s", vaultBaseUrl, vaultBaseSecretsKvPath, secretPath)
		if request, err := http.NewRequest(http.MethodDelete, url, nil); err != nil {
		// create request error: propagate
		return StatusError{
		Code: http.StatusInternalServerError,
		Message: "Failed creating request",
		Err: err,
		}
		} else if token, err := getToken(); err != nil {
		// login error: propagate
		return StatusError{
		Message: "Login failed",
		Err: err,
		}
		} else {
		request.Header.Set("Content-Type", "application/json")
		request.Header.Set(ClientTokenHeader, token)
		if len(vaultNamespace) > 0 {
		request.Header.Set(NamespaceHeader, vaultNamespace)
		}

		log.Printf("... delete secret '%s' (DELETE %s)\n", secretPath, url)
		client := &http.Client{}
		if resp, err := client.Do(request); err != nil {
		// request error: propagate
		return StatusError{
		Code: http.StatusInternalServerError,
		Message: "Failed sending request",
		Err: err,
		}
		} else if resp.StatusCode != http.StatusNoContent {
		// http error: decode error body and throw error
		defer resp.Body.Close() // nolint:errcheck
		var errResp ErrorResponse
		// nolint
		json.NewDecoder(resp.Body).Decode(&errResp)

		return StatusError{
		Code: resp.StatusCode,
		Message: fmt.Sprintf("Vault server error on DELETE %s", url),
		Err: errResp,
		}
		} else {
		return nil
		}
		}
		_, err := doSecretRequest(http.MethodDelete, secretPath, "")
		return err
		}

		// splitWithEscaping slices s into all substrings separated by sep and returns a slice of the substrings between those separators.
		@@ -635,10 +581,18 @@ func getObj(obj map[string]interface{}, keyPath string) (string, error) {
		}
		}

		/**
		* This function returns the secret path from the request URL
		* ex: /api/secrets/my/secret/path -> my/secret/path
		*/
		func getSecretPath(re *http.Request) string {
		return re.URL.Path[13:] // remove "/api/secrets/"
		}

		/**
		* Check json format into request body
		*/
		func checkBody(request *http.Request) (int, []byte, error) {
		func checkJsonBody(request *http.Request) (int, []byte, error) {
		bodyStr, _ := io.ReadAll(request.Body)
		if len(bodyStr) <= 0 {
		return http.StatusBadRequest, nil, fmt.Errorf("request body must not be empty")
		@@ -691,6 +645,7 @@ func checkBody(request *http.Request) (int, []byte, error) {
		* Routes the incoming request depending on the Http method:
		* - GET /api/secrets/{path}
		* - PUT/POST /api/secrets/{path}
		* - PATCH /api/secrets/{path}
		* - DELETE /api/secrets/{path}
		*/
		func SecretsEndpoint(w http.ResponseWriter, r *http.Request) {
		@@ -699,6 +654,8 @@ func SecretsEndpoint(w http.ResponseWriter, r *http.Request) {
		GetSecret(w, r)
		case "POST", "PUT":
		UpdateSecret(w, r)
		case "PATCH":
		PatchSecret(w, r)
		case "DELETE":
		DeleteSecret(w, r)
		default:
		@@ -711,7 +668,7 @@ func SecretsEndpoint(w http.ResponseWriter, r *http.Request) {
		*/
		func GetSecret(writer http.ResponseWriter, request *http.Request) {
		writer.Header().Set("Content-Type", "text/plain")
		secretPath := request.URL.Path[13:]
		secretPath := getSecretPath(request)
		keyPath := request.URL.Query().Get("field")

		if secret, err := getSecret(secretPath, keyPath); err != nil {
		@@ -730,9 +687,9 @@ func GetSecret(writer http.ResponseWriter, request *http.Request) {
		*/
		func UpdateSecret(writer http.ResponseWriter, request *http.Request) {
		writer.Header().Set("Content-Type", "application/json")
		secretPath := request.URL.Path[13:]
		secretPath := getSecretPath(request)

		if status, body, err := checkBody(request); err != nil {
		if status, body, err := checkJsonBody(request); err != nil {
		log.Printf("%d %s", status, err)
		http.Error(writer, err.Error(), status)
		} else {
		@@ -753,11 +710,58 @@ func UpdateSecret(writer http.ResponseWriter, request *http.Request) {
		}
		}

		/**
		* PATCH /api/secrets endpoint function
		*/
		func PatchSecret(writer http.ResponseWriter, request *http.Request) {
		writer.Header().Set("Content-Type", "application/json")
		secretPath := getSecretPath(request)

		var body string
		if keyPath := request.URL.Query().Get("field"); len(keyPath) > 0 {
		// field is specified: use it as key and convert body to JSON string
		bodyStr, _ := io.ReadAll(request.Body)
		if len(bodyStr) <= 0 {
		http.Error(writer, "Request body must not be empty", http.StatusBadRequest)
		return
		}
		bodyMap := map[string]interface{}{keyPath: string(bodyStr)}
		if bodyJson, err := json.Marshal(bodyMap); err != nil {
		http.Error(writer, fmt.Sprintf("Invalid JSON body: %s", err.Error()), http.StatusBadRequest)
		return
		} else {
		body = string(bodyJson)
		}
		} else if status, bytes, err := checkJsonBody(request); err == nil {
		// no field specified: use body as JSON string
		body = string(bytes)
		} else {
		log.Printf("%d %s", status, err)
		http.Error(writer, err.Error(), status)
		return
		}

		if createResp, err := doPatchSecret(secretPath, string(body)); err != nil {
		http.Error(writer, err.Error(), err.(StatusError).Status())
		} else {
		delete(path2secret, secretPath)
		createJson, err := json.Marshal(createResp)
		if err != nil {
		http.Error(writer, err.Error(), http.StatusInternalServerError)
		return
		}
		writer.WriteHeader(http.StatusOK)
		if _, err = writer.Write([]byte(createJson)); err != nil {
		log.Println("Error while sending response")
		}
		}
		}

		/**
		* DELETE /api/secrets endpoint function
		*/
		func DeleteSecret(writer http.ResponseWriter, request *http.Request) {
		secretPath := request.URL.Path[13:]
		secretPath := getSecretPath(request)

		if err := doDeleteSecret(secretPath); err != nil {
		http.Error(writer, err.Error(), err.(StatusError).Status())

cmd/vault_service/internal/vault_model.go

+2 −22

Original line number	Diff line number	Diff line
		@@ -100,26 +100,6 @@ type LoginResponse struct {
		Auth *Authentication `json:"auth"`
		}

		/**
		* See: https://www.vaultproject.io/api-docs/secret/kv/kv-v1#read-secret
		*/
		type GetSecretResponse struct {
		Metadata map[string]interface{} `json:"metadata"`
		WrapInfo map[string]string `json:"wrap_info"`
		LeaseDuration int64 `json:"lease_duration"`
		LeaseId string `json:"lease_id"`
		RequestId string `json:"request_id"`
		Warnings []string `json:"warnings"`
		Renewable bool `json:"renewable"`
		Data map[string]interface{} `json:"data"`
		}

		type UpdateSecretResponse struct {
		WrapInfo map[string]string `json:"wrap_info"`
		LeaseDuration int64 `json:"lease_duration"`
		LeaseId string `json:"lease_id"`
		RequestId string `json:"request_id"`
		Warnings []string `json:"warnings"`
		Renewable bool `json:"renewable"`
		type DataResponse struct {
		Data map[string]interface{} `json:"data"`
		}

cmd/vault_service/internal/vault_model_test.go

+2 −2

Original line number	Diff line number	Diff line
		@@ -94,7 +94,7 @@ func Test_get_secret_response_unmarshalling(t *testing.T) {
		"lease_id": "12345",
		"renewable": false
		}`
		var getSecretResp GetSecretResponse
		var getSecretResp DataResponse
		if err := json.Unmarshal([]byte(response), &getSecretResp); err != nil {
		t.Fatalf("Error while unmarshalling message: %v", err)
		} else if getSecretResp.Data["foo"] != "bar" {
		@@ -115,7 +115,7 @@ func Test_update_secret_response_unmarshalling(t *testing.T) {
		"lease_id": "",
		"renewable": false
		}`
		var updateSecretResp UpdateSecretResponse
		var updateSecretResp DataResponse
		if err := json.Unmarshal([]byte(response), &updateSecretResp); err != nil {
		t.Fatalf("Error while unmarshalling message: %v", err)
		} else if updateSecretResp.Data["created_time"] != "2021-11-10T10:11:12.400489065Z" {