From 362d1fcc2245c74513f35f090510d7365f667a69 Mon Sep 17 00:00:00 2001 From: Marcelo Correia Date: Thu, 2 Aug 2018 18:38:04 +1000 Subject: [PATCH] more goodies --- Makefile | 97 ++++++---- README.md | 146 +++++++++++++- ci/pipeline.yml | 24 ++- env.mk | 17 -- glide.lock | 23 ++- glide.yaml | 4 + main.go | 27 ++- pipeline.mk | 17 -- template-engine/custom-functions.go | 58 ++++++ template-engine/fileutils.go | 15 +- template-engine/string-utils.go | 1 - template-engine/template-engine.go | 181 +++++------------- template-engine/template-engine_test.go | 37 ++-- .../base/.templates/defaults.yml | 10 - template-engine/test_fixtures/v3.yaml | 10 + .../test_fixtures/variables.tfvars | 5 + update-brew.sh | 33 ++++ 17 files changed, 460 insertions(+), 245 deletions(-) create mode 100644 template-engine/custom-functions.go delete mode 100644 template-engine/test_fixtures/base/.templates/defaults.yml create mode 100644 template-engine/test_fixtures/v3.yaml create mode 100644 template-engine/test_fixtures/variables.tfvars create mode 100755 update-brew.sh diff --git a/Makefile b/Makefile index f03355c..88a3526 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,34 @@ -include env.mk pipeline.mk go.mk +APP_NAME := go-template-engine +GITHUB_USER := marcelocorreia +GOARCH :=amd64 +GOOS := darwin +GOPATH := /go +NAMESPACE := github.com/marcelocorreia +OUTPUT_FILE := ./bin/$(APP_NAME) +REPO_NAME := $(APP_NAME) +REPO_URL := git@github.com:$(GITHUB_USER)/$(APP_NAME).git +TEST_OUTPUT_DIR := tmp +#VERSION := $(shell make get-last-release) +WORKDIR := $(GOPATH)/src/$(NAMESPACE)/$(REPO_NAME) +HOMEBREW_REPO := git@github.com:marcelocorreia/homebrew-taps.git +HOMEBREW_BINARY := dist/$(APP_NAME)-darwin-amd64-$(VERSION).zip +#HOMEBREW_BINARY_SUM := $(shell shasum -a 256 $(HOMEBREW_BINARY) | awk '{print $$1}') +HOMEBREW_REPO_PATH ?= /Users/marcelo/IdeaProjects/tardis/homebrew-taps +DOCS_DIR := docs + +include go.mk + + + +pipeline: + fly -t main set-pipeline \ + -n -p $(APP_NAME) \ + -c ./ci/pipeline.yml \ + -l $(HOME)/.ssh/ci-credentials.yml \ + -l ci/properties.yml + + fly -t main unpause-pipeline -p $(APP_NAME) +.PHONY: pipeline test-full: clean_docs _docs-check cover-tests cover-out cover-html cover-cleanup @@ -20,7 +50,7 @@ build: $(call build,GOOS=$(GOOS) GOARCH=$(GOARCH),$(APP_NAME)) define build - $1 go build -o ./bin/$(APP_NAME) -ldflags "-X main.VERSION=dev" -v + $1 go build -o ./bin/$(APP_NAME) -ldflags "-X main.VERSION=dev-mc2" -v ./main.go endef _validate-version: @@ -38,14 +68,6 @@ GITHUB_USER := marcelocorreia get-last-release: @curl -s https://api.github.com/repos/$(GITHUB_USER)/$(APP_NAME)/tags | jq ".[]|.name" | head -n1 | sed 's/\"//g' | sed 's/v*//g' -homebrew-tap: - go-template-engine \ - --source ci/go-template-engine.rb \ - --var dist_file=dist/go-template-engine-darwin-amd64-1.39.0.zip \ - --var version=1.39.0 \ - --var hash_sum=123 \ - > /Users/marcelo/IdeaProjects/tardis/homebrew-taps/go-template-engine.rb - get-version: @git checkout origin/version -- version && \ @@ -55,34 +77,43 @@ get-version: _docs-check: @[ -f $(DOCS_DIR) ] && echo $(DOCS_DIR) folder found || mkdir -p $(DOCS_DIR) -concourse-pull: - cd ci && docker-compose pull -concourse-up: - cd ci && CONCOURSE_EXTERNAL_URL=http://localhost:8080 docker-compose up -d -concourse-down: - cd ci && docker-compose down +_validate-app-name: +ifndef APP_NAME + $(error APP_NAME is required) +endif -concourse-stop: - cd ci && docker-compose stop +update_brew: _validate-app-name + ./update-brew.sh $(APP_NAME) -concourse-start: - cd ci && docker-compose start +git-push: + git add . ; git commit -m "updating pipeline"; git push -concourse-logs: - cd ci && docker-compose logs -f +pipeline-full: git-push pipeline -concourse-keys: - @[ -f ./ci/keys ] && echo ./ci/keys folder found || $(call create-concourse-keys) +_prepare: + @echo $(GOPATH) - $(shell pwd) + @mkdir -p /go/src/$(NAMESPACE)/$(APP_NAME)/dist + @cp -R * /go/src/$(NAMESPACE)/$(APP_NAME)/ + @$(call ci_make,deps) -define create-concourse-keys - echo "Creating Concourse keys" - mkdir -p ./ci/keys/web ./ci/keys/worker; - ssh-keygen -t rsa -f ./ci/keys/web/tsa_host_key -N '' - ssh-keygen -t rsa -f ./ci/keys/web/session_signing_key -N '' - ssh-keygen -t rsa -f ./ci/keys/worker/worker_key -N '' - cp ./ci/keys/worker/worker_key.pub ./ci/keys/web/authorized_worker_keys - cp ./ci/keys/web/tsa_host_key.pub ./ci/keys/worker -endef +_build: + @$(call ci_make,lint build GOOS=linux) + +_test: + @$(call ci_make, test GOOS=linux) +_release: _validate-version + @$(call ci_make,release) + pwd + cp $(GOPATH)/src/$(NAMESPACE)/$(APP_NAME)/dist/*zip ../output/ +define ci_make + echo "" + echo "*** $1::Begin ***" + cd $(GOPATH)/src/$(NAMESPACE)/$(APP_NAME) && \ + make $1 + echo "*** $1::End ***" + echo "" + cd - +endef diff --git a/README.md b/README.md index 6da3d55..263de7e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,8 @@ Based on Golang templates text SDK. ## TLDR; - +- Added support to [HCL](https://github.com/hashicorp/hcl) formart for variables file input +- Added support to all [Masterminds Sprig](https://github.com/Masterminds/sprig) functions - Added static file include. Tag {{staticInclude "path/to/file.txt"}} - Added replace tag. Tag {{replace .var "FROM_THIS" "TO_THIS"}} - Accepts JSON and YAML variables files @@ -19,6 +20,146 @@ Based on Golang templates text SDK. - Can be extended - ```$> go get github.com/marcelocorreia/go-template-engine/template-engine``` +### Custom functions +|Function|Source|Desc| +|--------|------|----| +|abbrev|[spring](https://github.com/Masterminds/sprig)|na| +|abbrevboth|[spring](https://github.com/Masterminds/sprig)|na| +|add|[spring](https://github.com/Masterminds/sprig)|na| +|add1|[spring](https://github.com/Masterminds/sprig)|na| +|ago|[spring](https://github.com/Masterminds/sprig)|na| +|append|[spring](https://github.com/Masterminds/sprig)|na| +|atoi|[spring](https://github.com/Masterminds/sprig)|na| +|b32dec|[spring](https://github.com/Masterminds/sprig)|na| +|b32enc|[spring](https://github.com/Masterminds/sprig)|na| +|b64dec|[spring](https://github.com/Masterminds/sprig)|na| +|b64enc|[spring](https://github.com/Masterminds/sprig)|na| +|base|[spring](https://github.com/Masterminds/sprig)|na| +|biggest|[spring](https://github.com/Masterminds/sprig)|na| +|buildCustomCert|[spring](https://github.com/Masterminds/sprig)|na| +|camelcase|[spring](https://github.com/Masterminds/sprig)|na| +|cat|[spring](https://github.com/Masterminds/sprig)|na| +|ceil|[spring](https://github.com/Masterminds/sprig)|na| +|clean|[spring](https://github.com/Masterminds/sprig)|na| +|coalesce|[spring](https://github.com/Masterminds/sprig)|na| +|compact|[spring](https://github.com/Masterminds/sprig)|na| +|contains|[spring](https://github.com/Masterminds/sprig)|na| +|date|[spring](https://github.com/Masterminds/sprig)|na| +|dateInZone|[spring](https://github.com/Masterminds/sprig)|na| +|dateModify|[spring](https://github.com/Masterminds/sprig)|na| +|date_in_zone|[spring](https://github.com/Masterminds/sprig)|na| +|date_modify|[spring](https://github.com/Masterminds/sprig)|na| +|default|[spring](https://github.com/Masterminds/sprig)|na| +|derivePassword|[spring](https://github.com/Masterminds/sprig)|na| +|dict|[spring](https://github.com/Masterminds/sprig)|na| +|dir|[spring](https://github.com/Masterminds/sprig)|na| +|div|[spring](https://github.com/Masterminds/sprig)|na| +|empty|[spring](https://github.com/Masterminds/sprig)|na| +|env|[spring](https://github.com/Masterminds/sprig)|na| +|expandenv|[spring](https://github.com/Masterminds/sprig)|na| +|ext|[spring](https://github.com/Masterminds/sprig)|na| +|fail|[spring](https://github.com/Masterminds/sprig)|na| +|first|[spring](https://github.com/Masterminds/sprig)|na| +|float64|[spring](https://github.com/Masterminds/sprig)|na| +|floor|[spring](https://github.com/Masterminds/sprig)|na| +|genCA|[spring](https://github.com/Masterminds/sprig)|na| +|genPrivateKey|[spring](https://github.com/Masterminds/sprig)|na| +|genSelfSignedCert|[spring](https://github.com/Masterminds/sprig)|na| +|genSignedCert|[spring](https://github.com/Masterminds/sprig)|na| +|has|[spring](https://github.com/Masterminds/sprig)|na| +|hasKey|[spring](https://github.com/Masterminds/sprig)|na| +|hasPrefix|[spring](https://github.com/Masterminds/sprig)|na| +|hasSuffix|[spring](https://github.com/Masterminds/sprig)|na| +|hello|[spring](https://github.com/Masterminds/sprig)|na| +|htmlDate|[spring](https://github.com/Masterminds/sprig)|na| +|htmlDateInZone|[spring](https://github.com/Masterminds/sprig)|na| +|indent|[spring](https://github.com/Masterminds/sprig)|na| +|initial|[spring](https://github.com/Masterminds/sprig)|na| +|initials|[spring](https://github.com/Masterminds/sprig)|na| +|int|[spring](https://github.com/Masterminds/sprig)|na| +|int64|[spring](https://github.com/Masterminds/sprig)|na| +|isAbs|[spring](https://github.com/Masterminds/sprig)|na| +|join|[spring](https://github.com/Masterminds/sprig)|na| +|keys|[spring](https://github.com/Masterminds/sprig)|na| +|kindIs|[spring](https://github.com/Masterminds/sprig)|na| +|kindOf|[spring](https://github.com/Masterminds/sprig)|na| +|last|[spring](https://github.com/Masterminds/sprig)|na| +|list|[spring](https://github.com/Masterminds/sprig)|na| +|lower|[spring](https://github.com/Masterminds/sprig)|na| +|max|[spring](https://github.com/Masterminds/sprig)|na| +|merge|[spring](https://github.com/Masterminds/sprig)|na| +|min|[spring](https://github.com/Masterminds/sprig)|na| +|mod|[spring](https://github.com/Masterminds/sprig)|na| +|mul|[spring](https://github.com/Masterminds/sprig)|na| +|nindent|[spring](https://github.com/Masterminds/sprig)|na| +|nospace|[spring](https://github.com/Masterminds/sprig)|na| +|now|[spring](https://github.com/Masterminds/sprig)|na| +|omit|[spring](https://github.com/Masterminds/sprig)|na| +|pick|[spring](https://github.com/Masterminds/sprig)|na| +|pluck|[spring](https://github.com/Masterminds/sprig)|na| +|plural|[spring](https://github.com/Masterminds/sprig)|na| +|prepend|[spring](https://github.com/Masterminds/sprig)|na| +|push|[spring](https://github.com/Masterminds/sprig)|na| +|quote|[spring](https://github.com/Masterminds/sprig)|na| +|randAlpha|[spring](https://github.com/Masterminds/sprig)|na| +|randAlphaNum|[spring](https://github.com/Masterminds/sprig)|na| +|randAscii|[spring](https://github.com/Masterminds/sprig)|na| +|randNumeric|[spring](https://github.com/Masterminds/sprig)|na| +|regexFind|[spring](https://github.com/Masterminds/sprig)|na| +|regexFindAll|[spring](https://github.com/Masterminds/sprig)|na| +|regexMatch|[spring](https://github.com/Masterminds/sprig)|na| +|regexReplaceAll|[spring](https://github.com/Masterminds/sprig)|na| +|regexReplaceAllLiteral|[spring](https://github.com/Masterminds/sprig)|na| +|regexSplit|[spring](https://github.com/Masterminds/sprig)|na| +|repeat|[spring](https://github.com/Masterminds/sprig)|na| +|replace|GTE|na| +|rest|[spring](https://github.com/Masterminds/sprig)|na| +|reverse|[spring](https://github.com/Masterminds/sprig)|na| +|round|[spring](https://github.com/Masterminds/sprig)|na| +|semver|[spring](https://github.com/Masterminds/sprig)|na| +|semverCompare|[spring](https://github.com/Masterminds/sprig)|na| +|set|[spring](https://github.com/Masterminds/sprig)|na| +|sha1sum|[spring](https://github.com/Masterminds/sprig)|na| +|sha256sum|[spring](https://github.com/Masterminds/sprig)|na| +|shuffle|[spring](https://github.com/Masterminds/sprig)|na| +|snakecase|[spring](https://github.com/Masterminds/sprig)|na| +|sortAlpha|[spring](https://github.com/Masterminds/sprig)|na| +|split|[spring](https://github.com/Masterminds/sprig)|na| +|splitList|[spring](https://github.com/Masterminds/sprig)|na| +|squote|[spring](https://github.com/Masterminds/sprig)|na| +|staticInclude|GTE|na| +|sub|[spring](https://github.com/Masterminds/sprig)|na| +|substr|[spring](https://github.com/Masterminds/sprig)|na| +|swapcase|[spring](https://github.com/Masterminds/sprig)|na| +|ternary|[spring](https://github.com/Masterminds/sprig)|na| +|title|[spring](https://github.com/Masterminds/sprig)|na| +|toDate|[spring](https://github.com/Masterminds/sprig)|na| +|toJson|[spring](https://github.com/Masterminds/sprig)|na| +|toPrettyJson|[spring](https://github.com/Masterminds/sprig)|na| +|toString|[spring](https://github.com/Masterminds/sprig)|na| +|toStrings|[spring](https://github.com/Masterminds/sprig)|na| +|trim|[spring](https://github.com/Masterminds/sprig)|na| +|trimAll|[spring](https://github.com/Masterminds/sprig)|na| +|trimPrefix|[spring](https://github.com/Masterminds/sprig)|na| +|trimSuffix|[spring](https://github.com/Masterminds/sprig)|na| +|trimall|[spring](https://github.com/Masterminds/sprig)|na| +|trunc|[spring](https://github.com/Masterminds/sprig)|na| +|tuple|[spring](https://github.com/Masterminds/sprig)|na| +|typeIs|[spring](https://github.com/Masterminds/sprig)|na| +|typeIsLike|[spring](https://github.com/Masterminds/sprig)|na| +|typeOf|[spring](https://github.com/Masterminds/sprig)|na| +|uniq|[spring](https://github.com/Masterminds/sprig)|na| +|unset|[spring](https://github.com/Masterminds/sprig)|na| +|until|[spring](https://github.com/Masterminds/sprig)|na| +|untilStep|[spring](https://github.com/Masterminds/sprig)|na| +|untitle|[spring](https://github.com/Masterminds/sprig)|na| +|upper|[spring](https://github.com/Masterminds/sprig)|na| +|uuidv4|[spring](https://github.com/Masterminds/sprig)|na| +|without|[spring](https://github.com/Masterminds/sprig)|na| +|wrap|[spring](https://github.com/Masterminds/sprig)|na| +|wrapWith|[spring](https://github.com/Masterminds/sprig)|na| + + ### Options ``` $> go-template-engine --help @@ -340,4 +481,5 @@ outJson, _ := engine.ParseTemplateFile("test_fixtures/bb.txt.tpl", varsJson) - [x] Recursive processing - [x] Custom Delimeters - [x] Static Include tag -- [x] Replace tag \ No newline at end of file +- [x] Replace tag +- [x] Extra functions(tons.... thanks to [Masterminds Spring](https://github.com/Masterminds/sprig) \ No newline at end of file diff --git a/ci/pipeline.yml b/ci/pipeline.yml index f961ee2..172732a 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -16,6 +16,14 @@ resources: private_key: {{git_private_key}} username: {{github_user}} + - name: git-repo-homebrew + type: git + source: + uri: {{git_repo_url}} + branch: dev + private_key: {{git_private_key}} + username: {{github_user}} + - name: app-resource-version type: semver source: @@ -38,8 +46,8 @@ resources: type: s3 source: bucket: {{s3_bucket}} - access_key_id: {{aws_access_key_id}} - secret_access_key: {{aws_secret_access_key}} + access_key_id: {{labs_aws_access_key_id}} + secret_access_key: {{labs_aws_secret_access_key}} region_name: ap-southeast-2 regexp: (.*).zip @@ -86,3 +94,15 @@ jobs: file: output/*linux*zip acl: public-read + - name: update-homebrew-tap + build_logs_to_retain: 10 + serial: true + plan: + - get: git-repo-homebrew + trigger: true + + - task: update + file: git-repo-master/ci/release.yml + params: + MAKE_TARGETS: _prepare deps _build _release + diff --git a/env.mk b/env.mk index 153f065..e69de29 100644 --- a/env.mk +++ b/env.mk @@ -1,17 +0,0 @@ -APP_NAME=go-template-engine -GITHUB_USER=marcelocorreia -GOARCH?=amd64 -GOOS?=darwin -GOPATH?=/go -NAMESPACE=github.com/marcelocorreia -OUTPUT_FILE=./bin/$(APP_NAME) -REPO_NAME=$(APP_NAME) -REPO_URL=git@github.com:$(GITHUB_USER)/$(APP_NAME).git -TEST_OUTPUT_DIR=tmp -VERSION=$(shell make get-version) -WORKDIR=$(GOPATH)/src/$(NAMESPACE)/$(REPO_NAME) -HOMEBREW_REPO=git@github.com:marcelocorreia/homebrew-taps.git -HOMEBREW_BINARY=dist/$(APP_NAME)-darwin-amd64-$(VERSION).zip -HOMEBREW_BINARY_SUM=$(shell shasum -a 256 $(HOMEBREW_BINARY) | awk '{print $$1}') -HOMEBREW_REPO_PATH?=/Users/marcelo/IdeaProjects/tardis/homebrew-taps -DOCS_DIR=docs diff --git a/glide.lock b/glide.lock index c5bfde2..c26391c 100644 --- a/glide.lock +++ b/glide.lock @@ -1,19 +1,33 @@ -hash: 4920f54d6bcf90d754a91078cd0d886842a78c17f036347c089dc7f3d63b65a7 -updated: 2018-01-07T18:58:27.424969252+11:00 +hash: 2a12e04f03bb607b3dadfc385f4ceed786b8860a4ab0a251341b41f3b0fa5e8c +updated: 2018-06-24T23:32:39.462796+10:00 imports: - name: github.com/alecthomas/template version: a0175ee3bccc567396460bf5acd36800cb10c49c repo: https://github.com/alecthomas/template + subpackages: + - parse - name: github.com/alecthomas/units version: 2efee857e7cfd4f3d0138cc3cbb1b4966962b93a repo: https://github.com/alecthomas/units +- name: github.com/aokoli/goutils + version: 3391d3790d23d03408670993e957e8f408993c34 - name: github.com/davecgh/go-spew version: 346938d642f2ec3594ed81d874461961cd0faa76 repo: https://github.com/davecgh/go-spew subpackages: - spew +- name: github.com/google/uuid + version: 064e2069ce9c359c118179501254f67d7d37ba24 - name: github.com/hashicorp/hcl version: 99ce73d4fe576449f7a689d4fc2b2ad09a86bdaa +- name: github.com/huandu/xstrings + version: 7bb0250b58e5c15670406e6f93ffda43281305b1 +- name: github.com/imdario/mergo + version: "" +- name: github.com/Masterminds/semver + version: 59c29afe1a994eacb71c833025ca7acf874bb1da +- name: github.com/Masterminds/sprig + version: 6b2a58267f6a8b1dc8e2eb5519b984008fa85e8c - name: github.com/pmezard/go-difflib version: 792786c7400a136282c1664665ae0a8db921c6c2 repo: https://github.com/pmezard/go-difflib @@ -24,6 +38,11 @@ imports: repo: https://github.com/stretchr/testify subpackages: - assert +- name: golang.org/x/crypto + version: 2c99acdd1e9b90d779ca23f632aad86af9909c62 + subpackages: + - pbkdf2 + - scrypt - name: gopkg.in/alecthomas/kingpin.v2 version: e9044be3ab2a8e11d4e1f418d12f0790d57e8d70 repo: https://gopkg.in/alecthomas/kingpin.v2 diff --git a/glide.yaml b/glide.yaml index bf55da3..2864fe7 100644 --- a/glide.yaml +++ b/glide.yaml @@ -28,3 +28,7 @@ import: version: a3f3340b5840cee44f372bddb5880fcbc419b46a repo: https://gopkg.in/yaml.v2 - package: github.com/hashicorp/hcl +- package: github.com/Masterminds/sprig + version: ^2.15.0 +- package: github.com/imdario/mergo + version: ^0.3.5 diff --git a/main.go b/main.go index 94d95a8..4c8131e 100644 --- a/main.go +++ b/main.go @@ -9,21 +9,30 @@ import ( ) var ( - app = kingpin.New("go-template-engine", "") + app = kingpin.New("go-template-engine", "go-template-engine") + templateFile = app.Flag("source", "Template Source File").Short('s').String() + headerTemplateFiles = app.Flag("header-sources", "Extra Source Files to append as HEADER to the main template before processing."+ + "Useful to preload embedded templates").Strings() + footerTemplateFiles = app.Flag("footer-sources", "Extra Source Files to append as FOOTER to the main template before processing."+ + "Useful to preload embedded templates").Strings() + templateVars = app.Flag("var", "Params & Variables. Example --var hey=ho --var lets=go").StringMap() - templateVarsFile = app.Flag("var-file", "Variables File").Strings() + templateVarsFile = app.Flag("var-file", "Variables File").String() templateIgnores = app.Flag("skip-parsing", "Includes file but skips parsing").Strings() templateExcludes = app.Flag("exclude", "Excludes File from template job").Strings() templateExcludesDir = app.Flag("exclude-dir", "Excludes directory from template job").Strings() templateFileOutput = app.Flag("output", "File output full path").Short('o').String() + delimLeft = app.Flag("delim-left", "Left Delimiter").Default("{{").String() delimRight = app.Flag("delim-right", "Right Delimiter").Default("}}").String() - versionFlag = app.Flag("version", "App Version").Short('v').Bool() + listCustomFunctions = app.Flag("list-custom-functions", "List Custom Commands").Short('c').Bool() VERSION string ) func main() { + app.Version(VERSION).VersionFlag.Short('v') + kingpin.CommandLine.HelpFlag.Short('h') if len(os.Args) <= 1 { kingpin.Usage() os.Exit(1) @@ -37,20 +46,20 @@ func main() { handleErrorExit(err, "Error Loading engine") } - if *versionFlag { - fmt.Println(VERSION) + if *listCustomFunctions { + engine.ListFuncs() os.Exit(0) } - var varsBundle interface{} var jobVars interface{} - - varsBundle, _ = engine.VariablesFileMerge(*templateVarsFile, *templateVars) - jobVars, err = engine.LoadVars(varsBundle.(string)) + jobVars, err = engine.LoadVars(*templateVarsFile) if err != nil { handleErrorExit(err, "Error:") } + for k, v := range *templateVars { + jobVars.(map[interface{}]interface{})[k] = v + } render(jobVars, engine) os.Exit(0) diff --git a/pipeline.mk b/pipeline.mk index 287e4c1..d69ec28 100644 --- a/pipeline.mk +++ b/pipeline.mk @@ -1,25 +1,8 @@ -fly-login: - fly -t dev login -n dev -c https://ci.correia.io - git-push: git add . ; git commit -m "updating pipeline"; git push pipeline-full: git-push pipeline -pipeline: - fly -t dev set-pipeline \ - -n -p $(APP_NAME) \ - -c ./ci/pipeline.yml \ - -l $(HOME)/.ssh/ci-credentials.yml \ - -l ci/properties.yml - - fly -t dev unpause-pipeline -p $(APP_NAME) -.PHONY: pipeline - -pipeline-destroy: - fly -t dev destroy-pipeline -p $(APP_NAME) -.PHONY: pipeline-destroy - _prepare: @echo $(GOPATH) - $(shell pwd) @mkdir -p /go/src/$(NAMESPACE)/$(APP_NAME)/dist diff --git a/template-engine/custom-functions.go b/template-engine/custom-functions.go new file mode 100644 index 0000000..e7f121d --- /dev/null +++ b/template-engine/custom-functions.go @@ -0,0 +1,58 @@ +package template_engine + +import ( + "io/ioutil" + "fmt" + "strings" + "reflect" + "sort" + "github.com/Masterminds/sprig" +) + +func (gte TemplateEngine) staticInclude(sourceFile string) (string) { + body, err := ioutil.ReadFile(sourceFile) + if err != nil { + return fmt.Sprintf("ERROR including file: %s\n", sourceFile) + } + return string(body) +} + +func (gte TemplateEngine) replace(input, from, to string) string { + return strings.Replace(input, from, to, -1) +} + +func (gte TemplateEngine) inList(needle interface{}, haystack []interface{}, ) bool { + for _, h := range haystack { + if reflect.DeepEqual(needle, h) { + return true + } + } + return false +} + +func (gte TemplateEngine) printf(pattern string, params ...string) (string) { + return fmt.Sprintf(pattern, params) +} + +func (gte TemplateEngine) ListFuncs() { + funcs := make([]string, 0, len(gte.Funcs)) + for k := range gte.Funcs { + funcs = append(funcs, k) + } + sort.Strings(funcs) + + for _, v := range funcs { + fmt.Println(v) + } +} + +func (gte *TemplateEngine) loadFuncs() { + for k, v := range sprig.GenericFuncMap() { + gte.Funcs[k] = v + } + + gte.Funcs["staticInclude"] = func(path string) string { return gte.staticInclude(path) } + gte.Funcs["replace"] = func(input, from, to string) string { return gte.replace(input, from, to) } + gte.Funcs["inList"] = func(needle interface{}, haystack []interface{}, ) bool { return gte.inList(needle, haystack) } + gte.Funcs["printf"] = func(pattern string, params ...string) (string) { return gte.printf(pattern, params...) } +} diff --git a/template-engine/fileutils.go b/template-engine/fileutils.go index 54746f6..0ffd11e 100644 --- a/template-engine/fileutils.go +++ b/template-engine/fileutils.go @@ -34,6 +34,11 @@ func CopyFile(source string, dest string) (err error) { return } +func IsDirectory(path string) (bool, error) { + fileInfo, err := os.Stat(path) + return fileInfo.IsDir(), err +} + func ListDir(dir string) ([]os.FileInfo) { files, err := ioutil.ReadDir(dir) if err != nil { @@ -121,10 +126,14 @@ func Exists(path string) (bool, error) { return true, err } -func CreateNewDirectoryIfNil(path string) (error) { - exists, _ := Exists(path) +func CreateNewDirectoryIfNil(path string) (bool, error) { + exists, err := Exists(path) + if err != nil { + return false, err + } if !exists { os.MkdirAll(path, 00750) + return true, nil } - return nil + return false, nil } diff --git a/template-engine/string-utils.go b/template-engine/string-utils.go index f14d917..8041b03 100644 --- a/template-engine/string-utils.go +++ b/template-engine/string-utils.go @@ -13,7 +13,6 @@ const ( ) func SecureRandomAlphaString(length int) string { - result := make([]byte, length) bufferSize := int(float64(length)*1.3) for i, j, randomBytes := 0, 0, []byte{}; i < length; j++ { diff --git a/template-engine/template-engine.go b/template-engine/template-engine.go index 0a06fa8..580282c 100644 --- a/template-engine/template-engine.go +++ b/template-engine/template-engine.go @@ -10,8 +10,9 @@ import ( "path/filepath" "strings" "text/template" - "bufio" "errors" + "github.com/Masterminds/sprig" + "github.com/hashicorp/hcl" ) var DELIMS = []string{"{{", "}}"} @@ -19,29 +20,21 @@ var DELIMS = []string{"{{", "}}"} type Engine interface { ParseTemplateFile(templateFile string, params interface{}) (string, error) ParseTemplateString(templateString string, params interface{}) (string, error) - VariablesFileMerge(varsFile []string, extra_vars map[string]string) (string, error) LoadVars(filePath string) (interface{}, error) ProcessDirectory(sourceDir string, targetDir string, params interface{}, dirExclusions []string, fileExclusions []string, fileIgnores []string) (error) - GetFileList(dir string, fullPath bool, dirExclusions []string, fileExclusions []string) ([]string, error) + GetFileList(dir string, dirExclusions []string, fileExclusions []string) ([]string, error) PrepareOutputDirectory(sourceDir string, targetDir string, exclusions []string) (error) + loadFuncs() + ListFuncs() staticInclude(sourceFile string) (string) replace(input, from, to string) string + inList(needle interface{}, haystack []interface{}, ) bool + printf(pattern string,params ...string)(string) } type TemplateEngine struct { Delims []string -} - -func (gte TemplateEngine) staticInclude(sourceFile string) (string) { - body, err := ioutil.ReadFile(sourceFile) - if err != nil { - return fmt.Sprintf("ERROR including file: %s\n", sourceFile) - } - return string(body) -} - -func (gte TemplateEngine) replace(input, from, to string) string { - return strings.Replace(input, from, to, -1) + Funcs map[string]interface{} } func GetEngine(delims ...string) (*TemplateEngine, error) { @@ -51,8 +44,9 @@ func GetEngine(delims ...string) (*TemplateEngine, error) { engine := TemplateEngine{ Delims: DELIMS, + Funcs: make(map[string]interface{}), } - + engine.loadFuncs() return &engine, nil } @@ -74,9 +68,10 @@ func (gte TemplateEngine) ParseTemplateString(templateString string, params inte funcMap := template.FuncMap{ "staticInclude": func(path string) string { return gte.staticInclude(path) }, "replace": func(input, from, to string) string { return gte.replace(input, from, to) }, + "inList": func(needle interface{}, haystack []interface{}, ) bool { return gte.inList(needle, haystack) }, } - t, err := template.New("letter").Delims(gte.Delims[0], gte.Delims[1]).Funcs(funcMap).Parse(templateString) + t, err := template.New("gte").Delims(gte.Delims[0], gte.Delims[1]).Funcs(funcMap).Funcs(sprig.GenericFuncMap()).Parse(templateString) if err != nil { return templateString, err } @@ -96,63 +91,19 @@ func (gte TemplateEngine) LoadVars(filePath string) (interface{}, error) { file, _ := ioutil.ReadFile(filePath) if strings.HasSuffix(filePath, ".json") { - err := json.Unmarshal(file, &varsFile) - fmt.Println(&varsFile) - if err != nil { + if err := json.Unmarshal(file, &varsFile); err != nil { return nil, err } } else if strings.HasSuffix(filePath, ".yaml") || strings.HasSuffix(filePath, ".yml") { - err := yaml.Unmarshal(file, &varsFile) - if err != nil { + if err := yaml.Unmarshal(file, &varsFile); err != nil { return nil, err } - } - return varsFile, nil -} - -func (gte TemplateEngine) VariablesFileMerge(varsFile []string, extra_vars map[string]string) (string, error) { - tmpFile, _ := ioutil.TempFile("/tmp", "vars") - - for _, file := range varsFile { - content, err := ioutil.ReadFile(file) - if err != nil { - return "", err - } - tmpFile.Write(content) - tmpFile.WriteString("\n") - } - - for k, v := range extra_vars { - tmpFile.WriteString(fmt.Sprintf("%s: %s\n", k, v)) - } - tmpFile.Close() - cleanFile, err := cleanYamlFile(tmpFile.Name()) - if err != nil { - return "", err - } - os.Remove(tmpFile.Name()) - CopyFile(cleanFile, cleanFile+".yml") - - return cleanFile + ".yml", nil -} - -func cleanYamlFile(file string) (string, error) { - tmpFile, err := ioutil.TempFile("/tmp", "vars") - if err != nil { - return "", err - } - inFile, _ := os.Open(file) - defer inFile.Close() - scanner := bufio.NewScanner(inFile) - scanner.Split(bufio.ScanLines) - - for scanner.Scan() { - line := scanner.Text() - if !strings.HasPrefix(line, "---") && len(line) > 0 { - tmpFile.WriteString(line + "\n") + } else if strings.HasSuffix(filePath, ".tf") || strings.HasSuffix(filePath, ".tfvars") { + if err := hcl.Unmarshal(file, &varsFile); err != nil { + return nil, err } } - return tmpFile.Name(), nil + return varsFile, nil } func (gte TemplateEngine) ProcessDirectory(sourceDir string, targetDir string, params interface{}, dirExclusions []string, fileExclusions []string, fileIgnores []string) (error) { @@ -160,105 +111,77 @@ func (gte TemplateEngine) ProcessDirectory(sourceDir string, targetDir string, p if err != nil { return err } - list, err := gte.GetFileList(sourceDir, false, dirExclusions, fileExclusions) + list, err := gte.GetFileList(sourceDir, dirExclusions, fileExclusions) if err != nil { return err } for _, f := range list { - sourceFile := fmt.Sprintf("%s/%s", sourceDir, f) - targetFile := fmt.Sprintf("%s/%s", targetDir, f) - var body string - baseName := filepath.Base(sourceFile) - if StringInSlice(baseName, fileIgnores) { - c, err := ioutil.ReadFile(sourceFile) + sourceFile := f + targetFile := strings.Replace(f, sourceDir, targetDir, -1) + isDir, err := IsDirectory(sourceFile) + if err != nil { + + } + if !isDir { + body, err := ioutil.ReadFile(sourceFile) if err != nil { - fmt.Println("Error Reading:", sourceFile) - return err + fmt.Println(sourceFile) } - body = string(c) - } else { - body, err = gte.ParseTemplateFile(sourceFile, params) + b, err := gte.ParseTemplateFile(sourceFile, params) if err != nil { fmt.Printf("File: %s can't be loaded as template,\n\tContent writen without modifications.\n\tPlease check the tags is case this is not correct.\n-----------------------------\n%s\n-----------------------------\n", sourceFile, body) } - } - err = output(body, targetFile) - if err != nil { - fmt.Println(err) - return err + if err := output(b, targetFile); err != nil { + return err + } + } } return nil } -func (gte TemplateEngine) GetFileList(dir string, fullPath bool, dirExclusions []string, fileExclusions []string) ([]string, error) { - var fileList *[]string - fileList = &[]string{} - files, err := ioutil.ReadDir(dir) - if err != nil { - return nil, err - } - for _, f := range files { - if !StringInSlice(f.Name(), dirExclusions) && !StringInSlice(f.Name(), fileExclusions) { - if info, err := os.Stat(dir + "/" + f.Name()); err == nil && info.IsDir() { - gte.getTempList(dir+"/"+f.Name(), fileList) - } else { - *fileList = append(*fileList, dir+"/"+f.Name()) - } - } - } - - if fullPath { - return *fileList, nil +func (gte TemplateEngine) GetFileList(dir string, dirExclusions []string, fileExclusions []string) ([]string, error) { + var files []string + exists, err := Exists(dir) + if err != nil || !exists { + return nil, errors.New(dir + "does not exist") } + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + files = append(files, path) + return nil + }) - list := []string{} - for _, f := range *fileList { - root := filepath.Base(dir) - list = append(list, strings.Replace(strings.Replace(f, dir, root, -1), root+"/", "", -1)) + if err != nil { + return nil, err } - return list, nil -} -func (gte TemplateEngine) getTempList(dir string, fileList *[]string) { - files, _ := ioutil.ReadDir(dir) - for _, f := range files { - if info, err := os.Stat(dir + "/" + f.Name()); err == nil && info.IsDir() { - gte.getTempList(dir+"/"+f.Name(), fileList) - } else { - *fileList = append(*fileList, dir+"/"+f.Name()) - } - } + return files, nil } func (gte TemplateEngine) PrepareOutputDirectory(sourceDir string, targetDir string, exclusions []string) (error) { - if targetDir == "" { return errors.New("output must be provided when source is a directory") } CreateNewDirectoryIfNil(targetDir) - files, err := ioutil.ReadDir(sourceDir) + files, err := gte.GetFileList(sourceDir, exclusions, exclusions) if err != nil { return err } for _, d := range files { - if !StringInSlice(d.Name(), exclusions) { - if info, err := os.Stat(sourceDir + "/" + d.Name()); err == nil && info.IsDir() { - CreateNewDirectoryIfNil(targetDir + "/" + d.Name()) - } + + if info, err := os.Stat(d); err == nil && info.IsDir() { + newDir := strings.Replace(d, sourceDir, targetDir, -1) + CreateNewDirectoryIfNil(newDir) } + } return nil } func output(out string, templateFileOutput string) (error) { - err := ioutil.WriteFile(templateFileOutput, []byte(out), 0755) - if err != nil { - return err - } - return nil + return ioutil.WriteFile(templateFileOutput, []byte(out), 0755) } diff --git a/template-engine/template-engine_test.go b/template-engine/template-engine_test.go index 621b11f..9a5d9aa 100644 --- a/template-engine/template-engine_test.go +++ b/template-engine/template-engine_test.go @@ -24,7 +24,10 @@ func TestParseTemplateString(t *testing.T) { assert.Contains(t, out, "Hey ho, let's go") fmt.Println("Finished Test with vars...\n") } - +func TestListFuncs(t *testing.T) { + engine, _ := template_engine.GetEngine() + engine.ListFuncs() +} func TestTemplateJson(t *testing.T) { engine, _ := template_engine.GetEngine(DEFAULT_DELIMS[0], DEFAULT_DELIMS[1]) fmt.Println("Running Test with JSON file...") @@ -52,26 +55,16 @@ func TestTemplateErrorJson(t *testing.T) { fmt.Println("Finished Testing throwing error...\n") } -func TestTemplateEngine_VariablesFileMerge(t *testing.T) { - engine, err := template_engine.GetEngine(DEFAULT_DELIMS[0], DEFAULT_DELIMS[1]) - assert.Nil(t, err) - out, err := engine.VariablesFileMerge([]string{"test_fixtures/bb.yml", "test_fixtures/combo1.yml"}, getParams()) - assert.Nil(t, err) - out, err = engine.VariablesFileMerge([]string{"test_fixtures/bb.yml", "/a/file/that/should/not/exist"}, getParams()) - assert.Error(t, err) - os.Remove(out) - out, err = engine.VariablesFileMerge([]string{"test_fixtures/bb-broken.yml"}, getParams()) - assert.Nil(t, err) -} func TestTemplateEngine_GetFileList(t *testing.T) { - dir := "/go/src/github.com/marcelocorreia/go-template-engine/template-engine" + //dir := "/go/src/github.com/marcelocorreia/go-template-engine/template-engine" + dir := "/go/src/github.com/marcelocorreia/badwolf-templates/templates/badwolf/terraform-stack" engine, _ := template_engine.GetEngine(DEFAULT_DELIMS[0], DEFAULT_DELIMS[1]) - ll, _ := engine.GetFileList(dir, true, []string{}, []string{}) + ll, _ := engine.GetFileList(dir, []string{}, []string{}) assert.True(t, len(ll) > 0) - _, err := engine.GetFileList("/a/dir/that/should/not/exist", true, []string{}, []string{}) - assert.Error(t, err) + //_, err := engine.GetFileList("/a/dir/that/should/not/exist", true, []string{}, []string{}) + //assert.Error(t, err) } func TestPrepareOutputDirectory(t *testing.T) { @@ -119,30 +112,33 @@ func TestTemplateEngine_LoadVars(t *testing.T) { vars, err = engine.LoadVars(dir + "/test_fixtures/bb-broken.json") assert.Nil(t, vars) assert.Error(t, err) + vars, err = engine.LoadVars(dir + "/test_fixtures/variables.tfvars") + assert.NotNil(t, vars) + assert.Nil(t, err) } func TestTemplateEngine_ProcessDirectory(t *testing.T) { engine, _ := template_engine.GetEngine(DEFAULT_DELIMS[0], DEFAULT_DELIMS[1]) dir, _ := os.Getwd() tmpDir := os.TempDir() - err := engine.ProcessDirectory(dir+"/test_fixtures/base", tmpDir, nil, []string{".templates"},nil,nil) + err := engine.ProcessDirectory(dir+"/test_fixtures/base", tmpDir, nil, nil,nil,nil) assert.Nil(t, err) os.RemoveAll(tmpDir) tmpDir = os.TempDir() - err = engine.ProcessDirectory(dir+"/test_fixtures/base", tmpDir, nil, []string{".templates"},nil,nil) + err = engine.ProcessDirectory(dir+"/test_fixtures/base", tmpDir, nil, nil,nil,nil) assert.Nil(t, err) os.RemoveAll(tmpDir) tmpDir = os.TempDir() - err = engine.ProcessDirectory(dir+"/test_fixtures/base", tmpDir, nil, []string{".templates"},nil,[]string{".variables.tfvars"}) + err = engine.ProcessDirectory(dir+"/test_fixtures/base", tmpDir, nil, nil,nil,nil) assert.Nil(t, err) exists,err:= template_engine.Exists(tmpDir+"/.variables.tfvars") assert.True(t,exists) os.RemoveAll(tmpDir) tmpDir = os.TempDir() - err = engine.ProcessDirectory(dir+"/test_fixtures/base", "/a/dir/that/should/not/exist", nil, []string{".templates"},nil,nil) + err = engine.ProcessDirectory(dir+"/test_fixtures/base", "/a/dir/that/should/not/exist", nil, nil,nil,nil) assert.Error(t, err) os.RemoveAll(tmpDir) @@ -193,3 +189,4 @@ func TestTemplateEngine_replace(t *testing.T) { fmt.Println(out) } + diff --git a/template-engine/test_fixtures/base/.templates/defaults.yml b/template-engine/test_fixtures/base/.templates/defaults.yml deleted file mode 100644 index c3c1a0f..0000000 --- a/template-engine/test_fixtures/base/.templates/defaults.yml +++ /dev/null @@ -1,10 +0,0 @@ -docker_user: '{{docker_user}}' -docker_password: '{{docker_password}}' -docker_mail: '{{docker_mail}}' -github_token: '{{github_token}}' -github_user: '{{github_user}}' -aws_access_key_id: '{{aws_access_key_id}}' -aws_secret_access_key: '{{aws_secret_access_key}}' -slack_webook: '{{slack_webook}}' -git_private_key: '{{git_private_key}}' -git_repo_url: '{{git_repo_url}}' \ No newline at end of file diff --git a/template-engine/test_fixtures/v3.yaml b/template-engine/test_fixtures/v3.yaml new file mode 100644 index 0000000..97fecd5 --- /dev/null +++ b/template-engine/test_fixtures/v3.yaml @@ -0,0 +1,10 @@ +--- +region: ap-southeast-2 +org: correia +application: some_app +stack: a_stack +state: + bucket: bucket + dynamodb_table: dynamodb_table + key: key + workspace_key_prefix: workspace_key_prefix \ No newline at end of file diff --git a/template-engine/test_fixtures/variables.tfvars b/template-engine/test_fixtures/variables.tfvars new file mode 100644 index 0000000..e058997 --- /dev/null +++ b/template-engine/test_fixtures/variables.tfvars @@ -0,0 +1,5 @@ +// Global +org = "correia" +application = "badwolf" +stack = "core" + diff --git a/update-brew.sh b/update-brew.sh new file mode 100755 index 0000000..f560f24 --- /dev/null +++ b/update-brew.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -e + +APP_NAME=$1 +API_HTTPS=https://api.github.com/repos +GITHUB_USER=marcelocorreia +VERSION=$(curl ${API_HTTPS}/${GITHUB_USER}/${APP_NAME}/tags | jq '.[0].name' | sed 's/\"//g') +OS_DETECTED=$(uname -s) +TMPDIR=/tmp/homebrew-taps-update +PKG_FILE=${TMPDIR}/${APP_NAME}-darwin-amd64-${VERSION}.zip +if [ ${OS_DETECTED} == "Darwin" ];then + SHA_CMD="shasum -a 256" +else + SHA_CMD="sha256sum" +fi + + +git clone git@github.com:marcelocorreia/homebrew-taps.git ${TMPDIR} + +curl https://github.com/marcelocorreia/${APP_NAME}/releases/download/${VERSION}/${APP_NAME}-darwin-amd64-${VERSION}.zip \ + -o ${PKG_FILE} -L + +SUM=$(${SHA_CMD} ${PKG_FILE} | awk {'print $1'}) + +echo ${SUM} + +go-template-engine -s ${TMPDIR}/go-template-engine.tpl --var version=${VERSION} --var sha256sum=${SUM} \ + -o ${TMPDIR}/${APP_NAME}.rb + +cd ${TMPDIR} +git add . +git commit -m "Updating ${APP_NAME} Release: ${VERSION}" +git push \ No newline at end of file