Commit dd19c54f authored by Pierre Smeyers's avatar Pierre Smeyers
Browse files

feat: variables substitution enhancements 

The former variables substitution mechanism function has been improved:
- INFO log when a substitution occurs
- WARN log when a variable is not valuated
- appropriate encoding is automatically determined

BREAKING CHANGE: Now the variables substitution mechanism
implements complete YAML string encoding.
That might break projects that used to workaround the former
implementation flaws.
parent e2b8b5ff

README.md

+11 −3
Original line number Diff line number Diff line
@@ -230,9 +230,17 @@ You may optionally provide your own variable files (in your project) that will b 
1. look for an env-specific `cf-vars-$environment_type.yml` (e.g. `cf-vars-staging.yml` for staging environment).

2. or default `cf-vars.yml`.



:warning: Your `cf-vars-$environment_type.yml` or `cf-vars.yml` files **may** contain variable patterns such as `${MY_SECRET}`.

If so, those patterns will be evaluated (replaced) with actual environment values. Beware that those values can be leaked by the `cf push` output.

Multiline variables must be surrounded by **double quotes** (`"`).

#### Variables substitution mechanism (in variable files)



While your scripts may freely use any of the available variables, your variable files (`cf-vars-$environment_type.yml` and `cf-vars.yml`) 

may use a **variables substitution** mechanism implemented by the template:



- Using the syntax `${VARIABLE_NAME}` or `%{VARIABLE_NAME}`.

  :warning: Curly braces (`{}`) are mandatory in the expression (`$VARIABLE_NAME` won't be processed).

- Each of those expressions will be **dynamically expanded** in your variable files with the variable value, right before being used.

- Variable substitution expressions **must be contained in double-quoted strings**.

  The substitution implementation takes care of escaping characters that need to be (double quote `"`, backslash `\`, tab `\t`, carriage return `\n` and line feed `\r`).

- Variable substitution can be prevented by appending `# nosubst` at the end of any line.



### Routes management

templates/gitlab-ci-cf.yml

+74 −5
Original line number Diff line number Diff line
@@ -533,9 +533,78 @@ stages: 
    done

  }

  

  function awkenvsubst() {

    # escapes '&' char in variables for gsub

    awk '{while(match($0,"[$%]{[^}]*}")) {var=substr($0,RSTART+2,RLENGTH-3);val=ENVIRON[var];gsub("&","\\\\&",val);gsub("[$%]{"var"}",val)}}1'

  function tbc_envsubst() {

    awk '

      BEGIN {

        count_replaced_lines = 0

        # ASCII codes

        for (i=0; i<=255; i++)

          char2code[sprintf("%c", i)] = i

      }

      # determine encoding (from env or from file extension)

      function encoding() {

        enc = ENVIRON["TBC_ENVSUBST_ENCODING"]

        if (enc != "")

          return enc

        if (match(FILENAME, /\.(json|yaml|yml)$/))

          return "jsonstr"

        return "raw"

      }

      # see: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent

      function uriencode(str) {

        len = length(str)

        enc = ""

        for (i=1; i<=len; i++) {

          c = substr(str, i, 1);

          if (index("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.!~*'\''()", c))

            enc = enc c

          else

            enc = enc "%" sprintf("%02X", char2code[c])

        }

        return enc

      }

      !/# *nosubst/ {

        orig_line = $0

        line = $0

        count_repl_in_line = 0

        # /!\ 3rd arg (match) not supported in BusyBox awk

        while (match(line, /[$%]\{([[:alnum:]_]+)\}/)) {

          expr_start = RSTART

          expr_len = RLENGTH

          # get var name

          var = substr(line, expr_start+2, expr_len-3)

          # get var value (from env)

          val = ENVIRON[var]

          # check variable is set

          if (val == "") {

            printf("[\033[1;93mWARN\033[0m] Environment variable \033[33;1m%s\033[0m is not set or empty\n", var) > "/dev/stderr"

          } else {

            enc = encoding()

            if (enc == "jsonstr") {

              gsub(/["\\]/, "\\\\&", val)

              gsub("\n", "\\n", val)

              gsub("\r", "\\r", val)

              gsub("\t", "\\t", val)

            } else if (enc == "uricomp") {

              val = uriencode(val)

            } else if (enc == "raw") {

            } else {

              printf("[\033[1;93mWARN\033[0m] Unsupported encoding \033[33;1m%s\033[0m: ignored\n", enc) > "/dev/stderr"

            }

          }

          # replace expression in line

          line = substr(line, 1, expr_start - 1) val substr(line, expr_start + expr_len)

          count_repl_in_line++

        }

        if (count_repl_in_line) {

          if (count_replaced_lines == 0)

            printf("[\033[1;94mINFO\033[0m] Variable expansion occurred in file \033[33;1m%s\033[0m:\n", FILENAME) > "/dev/stderr"

          count_replaced_lines++

          printf("> line %s: %s\n", NR, orig_line) > "/dev/stderr"

        }

        print line

      }

    ' "$@"

  }



  function pre_push() {
@@ -563,11 +632,11 @@ stages: 
    if [[ -f "${envvarfile}" ]]

    then

      log_info "--- variables file for env \\e[33;1m${environment_type}\\e[0m (\\e[33;1m${envvarfile}\\e[0m) found"

      awkenvsubst < "$envvarfile" >> "$targetvarfile"

      tbc_envsubst "$envvarfile" >> "$targetvarfile"

    elif [[ -f "${globvarfile}" ]]

    then

      log_info "--- global variables file (\\e[33;1m${globvarfile}\\e[0m) found"

      awkenvsubst < "$globvarfile" >> "$targetvarfile"

      tbc_envsubst "$globvarfile" >> "$targetvarfile"

    fi



    log_info "--- manifest variables:"