From 939177be6b30bd19dd3acfc1ae85690b83803234 Mon Sep 17 00:00:00 2001 From: rblundon Date: Mon, 14 Apr 2025 23:43:59 -0500 Subject: [PATCH] Squashed 'acm-hub-bootstrap/' content from commit 4b79b1a0 git-subtree-dir: acm-hub-bootstrap git-subtree-split: 4b79b1a0be6da2a9efc9ace8871b4762342da70b --- .ansible-lint | 18 ++ .github/dependabot.yml | 9 + .github/linters/.gitleaks.toml | 9 + .github/linters/.markdown-lint.yml | 6 + .github/workflows/ansible-lint.yml | 17 ++ .github/workflows/jsonschema.yaml | 49 ++++ .github/workflows/superlinter.yml | 44 +++ .github/workflows/update-metadata.yml | 23 ++ .gitignore | 9 + .gitleaks.toml | 1 + LICENSE | 202 ++++++++++++++ Makefile | 25 ++ README.md | 19 ++ ansible.cfg | 6 + ansible/site.yaml | 17 ++ charts/all/config-demo/Chart.yaml | 6 + .../config-demo/templates/config-demo-cm.yaml | 24 ++ .../templates/config-demo-deployment.yaml | 65 +++++ .../config-demo-external-secret.yaml | 18 ++ .../config-demo/templates/config-demo-is.yaml | 12 + .../templates/config-demo-route.yaml | 14 + .../templates/config-demo-svc.yaml | 17 ++ charts/all/config-demo/values.yaml | 14 + charts/all/hello-world/Chart.yaml | 6 + .../hello-world/templates/hello-world-cm.yaml | 23 ++ .../templates/hello-world-deployment.yaml | 58 ++++ .../templates/hello-world-route.yaml | 14 + .../templates/hello-world-svc.yaml | 17 ++ charts/all/hello-world/values.yaml | 4 + charts/region/.keep | 0 common/.ansible-lint | 21 ++ common/.github/dependabot.yml | 9 + common/.github/linters/.gitleaks.toml | 4 + common/.github/linters/.markdown-lint.yml | 6 + common/.github/workflows/pattern-sh-ci.yml | 48 ++++ common/.github/workflows/superlinter.yml | 44 +++ common/.gitignore | 13 + common/.gitleaks.toml | 1 + common/Changes.md | 153 +++++++++++ common/LICENSE | 202 ++++++++++++++ common/Makefile | 255 ++++++++++++++++++ common/README.md | 51 ++++ common/requirements.yml | 4 + common/scripts/deploy-pattern.sh | 27 ++ common/scripts/determine-main-clustergroup.sh | 16 ++ common/scripts/determine-pattern-name.sh | 15 ++ .../scripts/determine-secretstore-backend.sh | 15 ++ common/scripts/display-secrets-info.sh | 28 ++ common/scripts/load-k8s-secrets.sh | 18 ++ common/scripts/make-common-subtree.sh | 76 ++++++ common/scripts/manage-secret-app.sh | 53 ++++ common/scripts/manage-secret-namespace.sh | 28 ++ common/scripts/pattern-util.sh | 112 ++++++++ common/scripts/preview-all.sh | 25 ++ common/scripts/preview.sh | 118 ++++++++ common/scripts/process-secrets.sh | 19 ++ common/scripts/set-secret-backend.sh | 5 + common/scripts/vault-utils.sh | 30 +++ common/scripts/write-token-kubeconfig.sh | 18 ++ overrides/values-AWS.yaml | 26 ++ overrides/values-IBMCloud.yaml | 10 + pattern-metadata.yaml | 70 +++++ pattern.sh | 1 + tests/interop/README.md | 25 ++ tests/interop/__init__.py | 2 + tests/interop/conftest.py | 2 + tests/interop/create_ci_badge.py | 84 ++++++ tests/interop/requirements.txt | 6 + tests/interop/run_tests.sh | 36 +++ tests/interop/test_modify_web_content.py | 89 ++++++ .../interop/test_subscription_status_edge.py | 25 ++ tests/interop/test_subscription_status_hub.py | 27 ++ .../test_validate_edge_site_components.py | 72 +++++ .../test_validate_hub_site_components.py | 95 +++++++ values-4.18-hub.yaml | 6 + values-global.yaml | 12 + values-group-one.yaml | 108 ++++++++ values-hub.yaml | 171 ++++++++++++ values-secret.yaml.template | 38 +++ values-standalone.yaml | 61 +++++ 80 files changed, 3126 insertions(+) create mode 100644 .ansible-lint create mode 100644 .github/dependabot.yml create mode 100644 .github/linters/.gitleaks.toml create mode 100644 .github/linters/.markdown-lint.yml create mode 100644 .github/workflows/ansible-lint.yml create mode 100644 .github/workflows/jsonschema.yaml create mode 100644 .github/workflows/superlinter.yml create mode 100644 .github/workflows/update-metadata.yml create mode 100644 .gitignore create mode 120000 .gitleaks.toml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 ansible.cfg create mode 100644 ansible/site.yaml create mode 100644 charts/all/config-demo/Chart.yaml create mode 100644 charts/all/config-demo/templates/config-demo-cm.yaml create mode 100644 charts/all/config-demo/templates/config-demo-deployment.yaml create mode 100644 charts/all/config-demo/templates/config-demo-external-secret.yaml create mode 100644 charts/all/config-demo/templates/config-demo-is.yaml create mode 100644 charts/all/config-demo/templates/config-demo-route.yaml create mode 100644 charts/all/config-demo/templates/config-demo-svc.yaml create mode 100644 charts/all/config-demo/values.yaml create mode 100644 charts/all/hello-world/Chart.yaml create mode 100644 charts/all/hello-world/templates/hello-world-cm.yaml create mode 100644 charts/all/hello-world/templates/hello-world-deployment.yaml create mode 100644 charts/all/hello-world/templates/hello-world-route.yaml create mode 100644 charts/all/hello-world/templates/hello-world-svc.yaml create mode 100644 charts/all/hello-world/values.yaml create mode 100644 charts/region/.keep create mode 100644 common/.ansible-lint create mode 100644 common/.github/dependabot.yml create mode 100644 common/.github/linters/.gitleaks.toml create mode 100644 common/.github/linters/.markdown-lint.yml create mode 100644 common/.github/workflows/pattern-sh-ci.yml create mode 100644 common/.github/workflows/superlinter.yml create mode 100644 common/.gitignore create mode 120000 common/.gitleaks.toml create mode 100644 common/Changes.md create mode 100644 common/LICENSE create mode 100644 common/Makefile create mode 100644 common/README.md create mode 100644 common/requirements.yml create mode 100755 common/scripts/deploy-pattern.sh create mode 100755 common/scripts/determine-main-clustergroup.sh create mode 100755 common/scripts/determine-pattern-name.sh create mode 100755 common/scripts/determine-secretstore-backend.sh create mode 100755 common/scripts/display-secrets-info.sh create mode 100755 common/scripts/load-k8s-secrets.sh create mode 100755 common/scripts/make-common-subtree.sh create mode 100755 common/scripts/manage-secret-app.sh create mode 100755 common/scripts/manage-secret-namespace.sh create mode 100755 common/scripts/pattern-util.sh create mode 100755 common/scripts/preview-all.sh create mode 100755 common/scripts/preview.sh create mode 100755 common/scripts/process-secrets.sh create mode 100755 common/scripts/set-secret-backend.sh create mode 100755 common/scripts/vault-utils.sh create mode 100755 common/scripts/write-token-kubeconfig.sh create mode 100644 overrides/values-AWS.yaml create mode 100644 overrides/values-IBMCloud.yaml create mode 100644 pattern-metadata.yaml create mode 120000 pattern.sh create mode 100644 tests/interop/README.md create mode 100644 tests/interop/__init__.py create mode 100644 tests/interop/conftest.py create mode 100644 tests/interop/create_ci_badge.py create mode 100644 tests/interop/requirements.txt create mode 100755 tests/interop/run_tests.sh create mode 100644 tests/interop/test_modify_web_content.py create mode 100644 tests/interop/test_subscription_status_edge.py create mode 100644 tests/interop/test_subscription_status_hub.py create mode 100644 tests/interop/test_validate_edge_site_components.py create mode 100644 tests/interop/test_validate_hub_site_components.py create mode 100644 values-4.18-hub.yaml create mode 100644 values-global.yaml create mode 100644 values-group-one.yaml create mode 100644 values-hub.yaml create mode 100644 values-secret.yaml.template create mode 100644 values-standalone.yaml diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000..040b390 --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,18 @@ +# Vim filetype=yaml +--- +offline: false +#requirements: ansible/execution_environment/requirements.yml + +exclude_paths: + - .cache/ + - .github/ + - charts/ + - common/ + - tests/ + +# warn_list: +# - yaml +# - schema +# - experimental +# - risky-file-permissions +# - var-spacing diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..a175e66 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +--- +version: 2 +updates: + # Check for updates to GitHub Actions every week + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + diff --git a/.github/linters/.gitleaks.toml b/.github/linters/.gitleaks.toml new file mode 100644 index 0000000..3386909 --- /dev/null +++ b/.github/linters/.gitleaks.toml @@ -0,0 +1,9 @@ +[whitelist] +# As of v4, gitleaks only matches against filename, not path in the +# files directive. Leaving content for backwards compatibility. +files = [ + "ansible/plugins/modules/*.py", + "ansible/tests/unit/test_*.py", + "ansible/tests/unit/*.yaml", + "ansible/tests/unit/v2/*.yaml", +] diff --git a/.github/linters/.markdown-lint.yml b/.github/linters/.markdown-lint.yml new file mode 100644 index 0000000..a0bc47d --- /dev/null +++ b/.github/linters/.markdown-lint.yml @@ -0,0 +1,6 @@ +{ + "default": true, + "MD003": false, + "MD013": false, + "MD033": false +} \ No newline at end of file diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml new file mode 100644 index 0000000..c2b2981 --- /dev/null +++ b/.github/workflows/ansible-lint.yml @@ -0,0 +1,17 @@ +name: Ansible Lint # feel free to pick your own name + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + # Important: This sets up your GITHUB_WORKSPACE environment variable + - uses: actions/checkout@v4 + + - name: Lint Ansible Playbook + uses: ansible/ansible-lint-action@v6 + # Let's point it to the path + with: + path: "ansible/" diff --git a/.github/workflows/jsonschema.yaml b/.github/workflows/jsonschema.yaml new file mode 100644 index 0000000..64ce8f8 --- /dev/null +++ b/.github/workflows/jsonschema.yaml @@ -0,0 +1,49 @@ +--- +name: Verify json schema + +on: [push, pull_request] + +jobs: + jsonschema_tests: + name: Json Schema tests + strategy: + matrix: + python-version: [3.11] + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install check-jsonschema + + - name: Install yq + uses: chrisdickinson/setup-yq@latest + with: + yq-version: v4.30.7 + + - name: Verify secrets json schema against templates + run: | + cp ./values-secret.yaml.template ./values-secret.yaml + check-jsonschema --fill-defaults --schemafile https://raw.githubusercontent.com/validatedpatterns/rhvp.cluster_utils/refs/heads/main/roles/vault_utils/values-secrets.v2.schema.json values-secret.yaml + rm -f ./values-secret.yaml + + - name: Verify ClusterGroup values.schema.json against values-*yaml files + run: | + set -e; for i in values-hub.yaml values-group-one.yaml; do + echo "$i" + # disable shellcheck of single quotes in yq + # shellcheck disable=2016 + yq eval-all '. as $item ireduce ({}; . * $item )' values-global.yaml "$i" > tmp.yaml + check-jsonschema --fill-defaults --schemafile https://raw.githubusercontent.com/validatedpatterns/clustergroup-chart/refs/heads/main/values.schema.json tmp.yaml + rm -f tmp.yaml + done + diff --git a/.github/workflows/superlinter.yml b/.github/workflows/superlinter.yml new file mode 100644 index 0000000..03b6fff --- /dev/null +++ b/.github/workflows/superlinter.yml @@ -0,0 +1,44 @@ +--- +name: Super linter + +on: [push, pull_request] + +jobs: + build: + # Name the Job + name: Super linter + # Set the agent to run on + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + # Full git history is needed to get a proper list of changed files within `super-linter` + fetch-depth: 0 + + ################################ + # Run Linter against code base # + ################################ + - name: Lint Code Base + uses: super-linter/super-linter/slim@v7 + env: + VALIDATE_ALL_CODEBASE: true + DEFAULT_BRANCH: main + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # These are the validation we disable atm + VALIDATE_ANSIBLE: false + VALIDATE_BASH: false + VALIDATE_CHECKOV: false + VALIDATE_JSCPD: false + VALIDATE_JSON_PRETTIER: false + VALIDATE_MARKDOWN_PRETTIER: false + VALIDATE_KUBERNETES_KUBECONFORM: false + VALIDATE_PYTHON_PYLINT: false + VALIDATE_SHELL_SHFMT: false + VALIDATE_YAML: false + VALIDATE_YAML_PRETTIER: false + # VALIDATE_DOCKERFILE_HADOLINT: false + # VALIDATE_MARKDOWN: false + # VALIDATE_NATURAL_LANGUAGE: false + # VALIDATE_TEKTON: false diff --git a/.github/workflows/update-metadata.yml b/.github/workflows/update-metadata.yml new file mode 100644 index 0000000..39f5889 --- /dev/null +++ b/.github/workflows/update-metadata.yml @@ -0,0 +1,23 @@ +# This job requires a secret called DOCS_TOKEN which should be a PAT token +# that has the permissions described in: +# validatedpatterns/docs/.github/workflows/metadata-docs.yml@main +--- +name: Update docs pattern metadata + +on: + push: + paths: + - "pattern-metadata.yaml" + - ".github/workflows/update-metadata.yml" + +jobs: + update-metadata: + uses: validatedpatterns/docs/.github/workflows/metadata-docs.yml@main + permissions: # Workflow-level permissions + contents: read # Required for "read-all" + packages: write # Allows writing to packages + id-token: write # Allows creating OpenID Connect (OIDC) tokens + secrets: inherit + # For testing you can point to a different branch in the docs repository + # with: + # DOCS_BRANCH: "main" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3f3db95 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +*~ +*.swp +*.swo +values-secret* +.*.expected.yaml +pattern-vault.init +vault.init +super-linter.log +common/pattern-vault.init diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 120000 index 0000000..c05303b --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1 @@ +.github/linters/.gitleaks.toml \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..39d07b7 --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +.PHONY: default +default: help + +.PHONY: help +##@ Pattern tasks + +# No need to add a comment here as help is described in common/ +help: + @make -f common/Makefile MAKEFILE_LIST="Makefile common/Makefile" help + +%: + make -f common/Makefile $* + +.PHONY: install +install: operator-deploy post-install ## installs the pattern and loads the secrets + @echo "Installed" + +.PHONY: post-install +post-install: ## Post-install tasks + make load-secrets + @echo "Done" + +.PHONY: test +test: + @make -f common/Makefile PATTERN_OPTS="-f values-global.yaml -f values-hub.yaml" test diff --git a/README.md b/README.md new file mode 100644 index 0000000..e09fd81 --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +# Multicloud Gitops + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +[Live build status](https://validatedpatterns.io/ci/?pattern=mcgitops) + +## Start Here + +If you've followed a link to this repository, but are not really sure what it contains +or how to use it, head over to [Multicloud GitOps](https://validatedpatterns.io/patterns/multicloud-gitops/) +for additional context and installation instructions + +## Rationale + +The goal for this pattern is to: + +* Use a GitOps approach to manage hybrid and multi-cloud deployments across both public and private clouds. +* Enable cross-cluster governance and application lifecycle management. +* Securely manage secrets across the deployment. diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..516f8b8 --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,6 @@ +[defaults] +localhost_warning=False +retry_files_enabled=False +library=~/.ansible/plugins/modules:./ansible/plugins/modules:./common/ansible/plugins/modules:/usr/share/ansible/plugins/modules +roles_path=~/.ansible/roles:./ansible/roles:./common/ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles +filter_plugins=~/.ansible/plugins/filter:./ansible/plugins/filter:./common/ansible/plugins/filter:/usr/share/ansible/plugins/filter diff --git a/ansible/site.yaml b/ansible/site.yaml new file mode 100644 index 0000000..f0b7c28 --- /dev/null +++ b/ansible/site.yaml @@ -0,0 +1,17 @@ +# This is only needed for RHPDS +- name: MultiCloud-GitOps RHPDS bootstrap + hosts: localhost + connection: local + tasks: + # We cannot use .package or .dnf modules because python3 that is used comes + # from a virtualenv + - name: Launch the installation + ansible.builtin.command: ./pattern.sh make install + args: + chdir: "{{ lookup('env', 'PWD') }}" + register: output + changed_when: false + + - name: Print output of installation + ansible.builtin.debug: + msg: "{{ output }}" diff --git a/charts/all/config-demo/Chart.yaml b/charts/all/config-demo/Chart.yaml new file mode 100644 index 0000000..abe0f0b --- /dev/null +++ b/charts/all/config-demo/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +description: A Helm chart to build and deploy a use of remote configuration enabled by ACM and Vault +keywords: +- pattern +name: config-demo +version: 0.0.1 diff --git a/charts/all/config-demo/templates/config-demo-cm.yaml b/charts/all/config-demo/templates/config-demo-cm.yaml new file mode 100644 index 0000000..ac7fe99 --- /dev/null +++ b/charts/all/config-demo/templates/config-demo-cm.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-demo-configmap + labels: + app.kubernetes.io/instance: config-demo +data: + "index.html": |- + + + + + Config Demo + + +

+ Hub Cluster domain is '{{ .Values.global.hubClusterDomain }}'
+ Pod is running on Local Cluster Domain '{{ .Values.global.localClusterDomain }}'
+

+

+ The secret is secret +

+ + diff --git a/charts/all/config-demo/templates/config-demo-deployment.yaml b/charts/all/config-demo/templates/config-demo-deployment.yaml new file mode 100644 index 0000000..9deee70 --- /dev/null +++ b/charts/all/config-demo/templates/config-demo-deployment.yaml @@ -0,0 +1,65 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + application: config-demo + name: config-demo +spec: + replicas: 2 + revisionHistoryLimit: 3 + selector: + matchLabels: + deploymentconfig: config-demo + template: + metadata: + creationTimestamp: null + labels: + app: config-demo + deploymentconfig: config-demo + name: config-demo + spec: + containers: + - name: apache + image: registry.access.redhat.com/ubi8/httpd-24:1-226 + #imagePullPolicy: Always + ports: + - containerPort: 8080 + name: http + protocol: TCP + volumeMounts: + - mountPath: /var/www/html + name: config-demo-configmap + - mountPath: /var/www/html/secret + readOnly: true + name: config-demo-secret + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + livenessProbe: + httpGet: + path: /index.html + port: 8080 + scheme: HTTP + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /index.html + port: 8080 + scheme: HTTP + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + volumes: + - name: config-demo-configmap + configMap: + defaultMode: 438 + name: config-demo-configmap + - name: config-demo-secret + secret: + secretName: config-demo-secret diff --git a/charts/all/config-demo/templates/config-demo-external-secret.yaml b/charts/all/config-demo/templates/config-demo-external-secret.yaml new file mode 100644 index 0000000..0081dd8 --- /dev/null +++ b/charts/all/config-demo/templates/config-demo-external-secret.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: "external-secrets.io/v1beta1" +kind: ExternalSecret +metadata: + name: config-demo-secret + namespace: config-demo +spec: + refreshInterval: 15s + secretStoreRef: + name: {{ .Values.secretStore.name }} + kind: {{ .Values.secretStore.kind }} + target: + name: config-demo-secret + template: + type: Opaque + dataFrom: + - extract: + key: {{ .Values.configdemosecret.key }} diff --git a/charts/all/config-demo/templates/config-demo-is.yaml b/charts/all/config-demo/templates/config-demo-is.yaml new file mode 100644 index 0000000..6a1aea4 --- /dev/null +++ b/charts/all/config-demo/templates/config-demo-is.yaml @@ -0,0 +1,12 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: config-demo +spec: + lookupPolicy: + local: true + tags: + - name: registry.access.redhat.com/ubi8/httpd-24 + importPolicy: {} + referencePolicy: + type: Local diff --git a/charts/all/config-demo/templates/config-demo-route.yaml b/charts/all/config-demo/templates/config-demo-route.yaml new file mode 100644 index 0000000..85d2e38 --- /dev/null +++ b/charts/all/config-demo/templates/config-demo-route.yaml @@ -0,0 +1,14 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + app: config-demo + name: config-demo +spec: + port: + targetPort: 8080-tcp + to: + kind: Service + name: config-demo + weight: 100 + wildcardPolicy: None diff --git a/charts/all/config-demo/templates/config-demo-svc.yaml b/charts/all/config-demo/templates/config-demo-svc.yaml new file mode 100644 index 0000000..517c1aa --- /dev/null +++ b/charts/all/config-demo/templates/config-demo-svc.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: config-demo + name: config-demo +spec: + ports: + - name: 8080-tcp + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: config-demo + deploymentconfig: config-demo + sessionAffinity: None + type: ClusterIP diff --git a/charts/all/config-demo/values.yaml b/charts/all/config-demo/values.yaml new file mode 100644 index 0000000..2dda452 --- /dev/null +++ b/charts/all/config-demo/values.yaml @@ -0,0 +1,14 @@ +--- +secretStore: + name: vault-backend + kind: ClusterSecretStore + +configdemosecret: + key: secret/data/global/config-demo + +global: + hubClusterDomain: hub.example.com + localClusterDomain: region-one.example.com + +clusterGroup: + isHubCluster: true diff --git a/charts/all/hello-world/Chart.yaml b/charts/all/hello-world/Chart.yaml new file mode 100644 index 0000000..6c9611e --- /dev/null +++ b/charts/all/hello-world/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +description: A Helm chart to show a webserver and with no other dependencies +keywords: +- pattern +name: hello-world +version: 0.0.1 diff --git a/charts/all/hello-world/templates/hello-world-cm.yaml b/charts/all/hello-world/templates/hello-world-cm.yaml new file mode 100644 index 0000000..e59561c --- /dev/null +++ b/charts/all/hello-world/templates/hello-world-cm.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: hello-world-configmap + labels: + app.kubernetes.io/instance: hello-world +data: + "index.html": |- + + + + + Hello World + + +

Hello World!

+
+

+ Hub Cluster domain is '{{ .Values.global.hubClusterDomain }}'
+ Pod is running on Local Cluster Domain '{{ .Values.global.localClusterDomain }}'
+

+ + diff --git a/charts/all/hello-world/templates/hello-world-deployment.yaml b/charts/all/hello-world/templates/hello-world-deployment.yaml new file mode 100644 index 0000000..878ebf5 --- /dev/null +++ b/charts/all/hello-world/templates/hello-world-deployment.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + application: hello-world + name: hello-world +spec: + replicas: 1 + revisionHistoryLimit: 3 + selector: + matchLabels: + deploymentconfig: hello-world + template: + metadata: + labels: + app: hello-world + deploymentconfig: hello-world + name: hello-world + spec: + containers: + - name: apache + image: registry.access.redhat.com/ubi8/httpd-24:1-226 + #imagePullPolicy: Always + ports: + - containerPort: 8080 + name: http + protocol: TCP + volumeMounts: + - mountPath: /var/www/html + name: hello-world-configmap + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + livenessProbe: + httpGet: + path: /index.html + port: 8080 + scheme: HTTP + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /index.html + port: 8080 + scheme: HTTP + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + volumes: + - name: hello-world-configmap + configMap: + defaultMode: 438 + name: hello-world-configmap diff --git a/charts/all/hello-world/templates/hello-world-route.yaml b/charts/all/hello-world/templates/hello-world-route.yaml new file mode 100644 index 0000000..e321f9e --- /dev/null +++ b/charts/all/hello-world/templates/hello-world-route.yaml @@ -0,0 +1,14 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + app: hello-world + name: hello-world +spec: + port: + targetPort: 8080-tcp + to: + kind: Service + name: hello-world + weight: 100 + wildcardPolicy: None diff --git a/charts/all/hello-world/templates/hello-world-svc.yaml b/charts/all/hello-world/templates/hello-world-svc.yaml new file mode 100644 index 0000000..597f6d5 --- /dev/null +++ b/charts/all/hello-world/templates/hello-world-svc.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: hello-world + name: hello-world +spec: + ports: + - name: 8080-tcp + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: hello-world + deploymentconfig: hello-world + sessionAffinity: None + type: ClusterIP diff --git a/charts/all/hello-world/values.yaml b/charts/all/hello-world/values.yaml new file mode 100644 index 0000000..55083f7 --- /dev/null +++ b/charts/all/hello-world/values.yaml @@ -0,0 +1,4 @@ +--- +global: + hubClusterDomain: hub.example.com + localCluster: local.example.com diff --git a/charts/region/.keep b/charts/region/.keep new file mode 100644 index 0000000..e69de29 diff --git a/common/.ansible-lint b/common/.ansible-lint new file mode 100644 index 0000000..0522976 --- /dev/null +++ b/common/.ansible-lint @@ -0,0 +1,21 @@ +# Vim filetype=yaml +--- +offline: false +skip_list: + - name[template] # Allow Jinja templating inside task and play names + - template-instead-of-copy # Templated files should use template instead of copy + - yaml[line-length] # too long lines + - yaml[indentation] # Forcing lists to be always indented by 2 chars is silly IMO + - var-naming[no-role-prefix] # This would be too much churn for very little gain + - no-changed-when + - var-naming[no-role-prefix] # There are too many changes now and it would be too risky + +# ansible-lint gh workflow cannot find ansible.cfg hence fails to import vault_utils role +exclude_paths: + - ./ansible/playbooks/vault/vault.yaml + - ./ansible/playbooks/iib-ci/iib-ci.yaml + - ./ansible/playbooks/k8s_secrets/k8s_secrets.yml + - ./ansible/playbooks/process_secrets/process_secrets.yml + - ./ansible/playbooks/write-token-kubeconfig/write-token-kubeconfig.yml + - ./ansible/playbooks/process_secrets/display_secrets_info.yml + - ./ansible/roles/vault_utils/tests/test.yml diff --git a/common/.github/dependabot.yml b/common/.github/dependabot.yml new file mode 100644 index 0000000..a175e66 --- /dev/null +++ b/common/.github/dependabot.yml @@ -0,0 +1,9 @@ +--- +version: 2 +updates: + # Check for updates to GitHub Actions every week + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + diff --git a/common/.github/linters/.gitleaks.toml b/common/.github/linters/.gitleaks.toml new file mode 100644 index 0000000..9ad7434 --- /dev/null +++ b/common/.github/linters/.gitleaks.toml @@ -0,0 +1,4 @@ +[whitelist] +# As of v4, gitleaks only matches against filename, not path in the +# files directive. Leaving content for backwards compatibility. +files = [ ] diff --git a/common/.github/linters/.markdown-lint.yml b/common/.github/linters/.markdown-lint.yml new file mode 100644 index 0000000..a0bc47d --- /dev/null +++ b/common/.github/linters/.markdown-lint.yml @@ -0,0 +1,6 @@ +{ + "default": true, + "MD003": false, + "MD013": false, + "MD033": false +} \ No newline at end of file diff --git a/common/.github/workflows/pattern-sh-ci.yml b/common/.github/workflows/pattern-sh-ci.yml new file mode 100644 index 0000000..ed0e6a0 --- /dev/null +++ b/common/.github/workflows/pattern-sh-ci.yml @@ -0,0 +1,48 @@ +name: Run Bash Script on Multiple Distributions + +on: + push: + paths: + - "scripts/**" + - "Makefile" + branches: + - main + pull_request: + paths: + - "scripts/**" + - "Makefile" + +jobs: + run-script: + name: Run Bash Script + strategy: + matrix: + # Fedora is not an option yet + os: [ubuntu-latest, ubuntu-22.04] + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Install Podman on Ubuntu + if: contains(matrix.os, 'ubuntu') + run: | + sudo apt-get update + sudo apt-get install -y podman + + # Currently we do not do MacOSX as it is not free, maybe in the future + # - name: Install Podman on macOS + # if: contains(matrix.os, 'macos') + # run: | + # brew install podman + # podman machine init + # podman machine start + + - name: Verify Podman Installation + run: podman --version + + - name: Run pattern.sh script + run: | + export TARGET_BRANCH=main + ./scripts/pattern-util.sh make validate-origin diff --git a/common/.github/workflows/superlinter.yml b/common/.github/workflows/superlinter.yml new file mode 100644 index 0000000..03b6fff --- /dev/null +++ b/common/.github/workflows/superlinter.yml @@ -0,0 +1,44 @@ +--- +name: Super linter + +on: [push, pull_request] + +jobs: + build: + # Name the Job + name: Super linter + # Set the agent to run on + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + # Full git history is needed to get a proper list of changed files within `super-linter` + fetch-depth: 0 + + ################################ + # Run Linter against code base # + ################################ + - name: Lint Code Base + uses: super-linter/super-linter/slim@v7 + env: + VALIDATE_ALL_CODEBASE: true + DEFAULT_BRANCH: main + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # These are the validation we disable atm + VALIDATE_ANSIBLE: false + VALIDATE_BASH: false + VALIDATE_CHECKOV: false + VALIDATE_JSCPD: false + VALIDATE_JSON_PRETTIER: false + VALIDATE_MARKDOWN_PRETTIER: false + VALIDATE_KUBERNETES_KUBECONFORM: false + VALIDATE_PYTHON_PYLINT: false + VALIDATE_SHELL_SHFMT: false + VALIDATE_YAML: false + VALIDATE_YAML_PRETTIER: false + # VALIDATE_DOCKERFILE_HADOLINT: false + # VALIDATE_MARKDOWN: false + # VALIDATE_NATURAL_LANGUAGE: false + # VALIDATE_TEKTON: false diff --git a/common/.gitignore b/common/.gitignore new file mode 100644 index 0000000..454efc9 --- /dev/null +++ b/common/.gitignore @@ -0,0 +1,13 @@ +__pycache__/ +*.py[cod] +*~ +*.swp +*.swo +values-secret.yaml +.*.expected.yaml +.vscode +pattern-vault.init +pattern-vault.init.bak +super-linter.log +golang-external-secrets/Chart.lock +hashicorp-vault/Chart.lock diff --git a/common/.gitleaks.toml b/common/.gitleaks.toml new file mode 120000 index 0000000..c05303b --- /dev/null +++ b/common/.gitleaks.toml @@ -0,0 +1 @@ +.github/linters/.gitleaks.toml \ No newline at end of file diff --git a/common/Changes.md b/common/Changes.md new file mode 100644 index 0000000..c12f175 --- /dev/null +++ b/common/Changes.md @@ -0,0 +1,153 @@ +# Changes + +## Sep 24, 2024 + +* Ansible has been moved out of the common code tree, you must use a clustergroup chart that is >= 0.9.1 + +## Sep 6, 2024 + +* Most charts have been removed from the tree. To get the charts you now have to point to them + +## Sep 25, 2023 + +* Upgraded ESO to v0.9.5 + +## Aug 17, 2023 + +* Introduced support for multisource applications via .chart + .chartVersion + +## Jul 8, 2023 + +* Introduced a default of 20 for sync failures retries in argo applications (global override via global.options.applicationRetryLimit + and per-app override via .syncPolicy) + +## May 22, 2023 + +* Upgraded ESO to 0.8.2 +* *Important* we now use the newly blessed sso config for argo. This means that gitops < 1.8 are *unsupported* + +## May 18, 2023 + +* Introduce a EXTRA_HELM_OPTS env variable that will be passed to the helm invocations + +## April 21, 2023 + +* Added labels and annotation support to namespaces.yaml template + +## Apr 11, 2023 + +* Apply the ACM ocp-gitops-policy everywhere but the hub + +## Apr 7, 2023 + +* Moved to gitops-1.8 channel by default (stable is unmaintained and will be dropped starting with ocp-4.13) + +## March 20, 2023 + +* Upgraded ESO to 0.8.1 + +## February 9, 2023 + +* Add support for /values-.yaml and for /values--.yaml + +## January 29, 2023 + +* Stop extracting the HUB's CA via an imperative job running on the imported cluster. + Just use ACM to push the HUB's CA out to the managed clusters. + +## January 23, 2023 + +* Add initial support for running ESO on ACM-imported clusters + +## January 18, 2023 + +* Add validate-schema target + +## January 13, 2023 + +* Simplify the secrets paths when using argo hosted sites + +## January 10, 2023 + +* vaultPrefixes is now optional in the v2 secret spec and defaults to ["hub"] + +## December 9, 2022 + +* Dropped insecureUnsealVaultInsideCluster (and file_unseal) entirely. Now + vault is always unsealed via a cronjob in the cluster. It is recommended to + store the imperative/vaultkeys secret offline securely and then delete it. + +## December 8, 2022 + +* Removed the legacy installation targets: + `deploy upgrade legacy-deploy legacy-upgrade` + Patterns must now use the operator-based installation + +## November 29, 2022 + +* Upgraded vault-helm to 0.23.0 +* Enable vault-ssl by default + +## November 22, 2022 + +* Implemented a new format for the values-secret.yaml. Example can be found in examples/ folder +* Now the order of values-secret file lookup is the following: + 1. ~/values-secret-.yaml + 2. ~/values-secret.yaml + 3. /values-secret.yaml.template +* Add support for ansible vault encrypted values-secret files. You can now encrypt your values-secret file + at rest with `ansible-vault encrypt ~/values-secret.yaml`. When running `make load-secrets` if an encrypted + file is encountered the user will be prompted automatically for the password to decrypt it. + +## November 6, 2022 + +* Add support for /values--.yaml (e.g. /values-AWS-group-one.yaml) + +## October 28, 2022 + +* Updated vault helm chart to v0.22.1 and vault containers to 1.12.0 + +## October 25, 2022 + +* Updated External Secrets Operator to v0.6.0 +* Moved to -UBI based ESO containers + +## October 13, 2022 + +* Added global.clusterVersion as a new helm variable which represents the OCP + Major.Minor cluster version. By default now a user can add a + values--.yaml file to have specific cluster version + overrides (e.g. values-4.10-hub.yaml). Will need Validated Patterns Operator >= 0.0.6 + when deploying with the operator. Note: When using the ArgoCD Hub and spoke model, + you cannot have spokes with a different version of OCP than the hub. + +## October 4, 2022 + +* Extended the values-secret.yaml file to support multiple vault paths and re-wrote + the push_secrets feature as python module plugin. This requires the following line + in a pattern's ansible.cfg's '[defaults]' stanza: + + `library=~/.ansible/plugins/modules:./ansible/plugins/modules:./common/ansible/plugins/modules:/usr/share/ansible/plugins/modules` + +## October 3, 2022 + +* Restore the ability to install a non-default site: `make TARGET_SITE=mysite install` +* Revised tests (new output and filenames, requires adding new result files to Git) +* ACM 2.6 required for ACM-based managed sites +* Introduced global.clusterDomain template variable (without the `apps.` prefix) +* Removed the ability to send specific charts to another cluster, use hosted argo sites instead +* Added the ability to have the hub host `values-{site}.yaml` for spoke clusters. + + The following example would deploy the namespaces, subscriptions, and + applications defined in `values-group-one.yaml` to the `perth` cluster + directly from ArgoCD on the hub. + + ```yaml + managedClusterGroups: + - name: group-one + hostedArgoSites: + - name: perth + domain: perth1.beekhof.net + bearerKeyPath: secret/data/hub/cluster_perth + caKeyPath: secret/data/hub/cluster_perth_ca + ``` diff --git a/common/LICENSE b/common/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/common/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/common/Makefile b/common/Makefile new file mode 100644 index 0000000..84f6afc --- /dev/null +++ b/common/Makefile @@ -0,0 +1,255 @@ +NAME ?= $(shell basename "`pwd`") + +ifneq ($(origin TARGET_SITE), undefined) + TARGET_SITE_OPT=--set main.clusterGroupName=$(TARGET_SITE) +endif + +# This variable can be set in order to pass additional helm arguments from the +# the command line. I.e. we can set things without having to tweak values files +EXTRA_HELM_OPTS ?= + +# This variable can be set in order to pass additional ansible-playbook arguments from the +# the command line. I.e. we can set -vvv for more verbose logging +EXTRA_PLAYBOOK_OPTS ?= + +# INDEX_IMAGES=registry-proxy.engineering.redhat.com/rh-osbs/iib:394248 +# or +# INDEX_IMAGES=registry-proxy.engineering.redhat.com/rh-osbs/iib:394248,registry-proxy.engineering.redhat.com/rh-osbs/iib:394249 +INDEX_IMAGES ?= + +TARGET_ORIGIN ?= origin +# This is to ensure that whether we start with a git@ or https:// URL, we end up with an https:// URL +# This is because we expect to use tokens for repo authentication as opposed to SSH keys +TARGET_REPO=$(shell git ls-remote --get-url --symref $(TARGET_ORIGIN) | sed -e 's/.*URL:[[:space:]]*//' -e 's%^git@%%' -e 's%^https://%%' -e 's%:%/%' -e 's%^%https://%') +# git branch --show-current is also available as of git 2.22, but we will use this for compatibility +TARGET_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) + +UUID_FILE ?= ~/.config/validated-patterns/pattern-uuid +UUID_HELM_OPTS ?= + +# --set values always take precedence over the contents of -f +ifneq ("$(wildcard $(UUID_FILE))","") + UUID := $(shell cat $(UUID_FILE)) + UUID_HELM_OPTS := --set main.analyticsUUID=$(UUID) +endif + +# Set the secret name *and* its namespace when deploying from private repositories +# The format of said secret is documented here: https://argo-cd.readthedocs.io/en/stable/operator-manual/declarative-setup/#repositories +TOKEN_SECRET ?= +TOKEN_NAMESPACE ?= + +ifeq ($(TOKEN_SECRET),) + HELM_OPTS=-f values-global.yaml --set main.git.repoURL="$(TARGET_REPO)" --set main.git.revision=$(TARGET_BRANCH) $(TARGET_SITE_OPT) $(UUID_HELM_OPTS) $(EXTRA_HELM_OPTS) +else + # When we are working with a private repository we do not escape the git URL as it might be using an ssh secret which does not use https:// + TARGET_CLEAN_REPO=$(shell git ls-remote --get-url --symref $(TARGET_ORIGIN)) + HELM_OPTS=-f values-global.yaml --set main.tokenSecret=$(TOKEN_SECRET) --set main.tokenSecretNamespace=$(TOKEN_NAMESPACE) --set main.git.repoURL="$(TARGET_CLEAN_REPO)" --set main.git.revision=$(TARGET_BRANCH) $(TARGET_SITE_OPT) $(UUID_HELM_OPTS) $(EXTRA_HELM_OPTS) +endif + +# Helm does the right thing and fetches all the tags and detects the newest one +PATTERN_INSTALL_CHART ?= oci://quay.io/hybridcloudpatterns/pattern-install + +##@ Pattern Common Tasks + +.PHONY: help +help: ## This help message + @echo "Pattern: $(NAME)" + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^(\s|[a-zA-Z_0-9-])+:.*?##/ { printf " \033[36m%-35s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +# Makefiles in the individual patterns should call these targets explicitly +# e.g. from industrial-edge: make -f common/Makefile show +.PHONY: show +show: ## show the starting template without installing it + helm template $(PATTERN_INSTALL_CHART) --name-template $(NAME) $(HELM_OPTS) + +preview-all: ## (EXPERIMENTAL) Previews all applications on hub and managed clusters + @echo "NOTE: This is just a tentative approximation of rendering all hub and managed clusters templates" + @common/scripts/preview-all.sh $(TARGET_REPO) $(TARGET_BRANCH) + +preview-%: + $(eval CLUSTERGROUP ?= $(shell yq ".main.clusterGroupName" values-global.yaml)) + @common/scripts/preview.sh $(CLUSTERGROUP) $* $(TARGET_REPO) $(TARGET_BRANCH) + +.PHONY: operator-deploy +operator-deploy operator-upgrade: validate-prereq validate-origin validate-cluster ## runs helm install + @common/scripts/deploy-pattern.sh $(NAME) $(PATTERN_INSTALL_CHART) $(HELM_OPTS) + +.PHONY: uninstall +uninstall: ## runs helm uninstall + $(eval CSV := $(shell oc get subscriptions -n openshift-operators openshift-gitops-operator -ojsonpath={.status.currentCSV})) + helm uninstall $(NAME) + @oc delete csv -n openshift-operators $(CSV) + +.PHONY: load-secrets +load-secrets: ## loads the secrets into the backend determined by values-global setting + common/scripts/process-secrets.sh $(NAME) + +.PHONY: legacy-load-secrets +legacy-load-secrets: ## loads the secrets into vault (only) + common/scripts/vault-utils.sh push_secrets $(NAME) + +.PHONY: secrets-backend-vault +secrets-backend-vault: ## Edits values files to use default Vault+ESO secrets config + common/scripts/set-secret-backend.sh vault + common/scripts/manage-secret-app.sh vault present + common/scripts/manage-secret-app.sh golang-external-secrets present + common/scripts/manage-secret-namespace.sh validated-patterns-secrets absent + @git diff --exit-code || echo "Secrets backend set to vault, please review changes, commit, and push to activate in the pattern" + +.PHONY: secrets-backend-kubernetes +secrets-backend-kubernetes: ## Edits values file to use Kubernetes+ESO secrets config + common/scripts/set-secret-backend.sh kubernetes + common/scripts/manage-secret-namespace.sh validated-patterns-secrets present + common/scripts/manage-secret-app.sh vault absent + common/scripts/manage-secret-app.sh golang-external-secrets present + @git diff --exit-code || echo "Secrets backend set to kubernetes, please review changes, commit, and push to activate in the pattern" + +.PHONY: secrets-backend-none +secrets-backend-none: ## Edits values files to remove secrets manager + ESO + common/scripts/set-secret-backend.sh none + common/scripts/manage-secret-app.sh vault absent + common/scripts/manage-secret-app.sh golang-external-secrets absent + common/scripts/manage-secret-namespace.sh validated-patterns-secrets absent + @git diff --exit-code || echo "Secrets backend set to none, please review changes, commit, and push to activate in the pattern" + +.PHONY: load-iib +load-iib: ## CI target to install Index Image Bundles + @set -e; if [ x$(INDEX_IMAGES) != x ]; then \ + ansible-playbook $(EXTRA_PLAYBOOK_OPTS) rhvp.cluster_utils.iib_ci; \ + else \ + echo "No INDEX_IMAGES defined. Bailing out"; \ + exit 1; \ + fi + +.PHONY: token-kubeconfig +token-kubeconfig: ## Create a local ~/.kube/config with password (not usually needed) + common/scripts/write-token-kubeconfig.sh + +##@ Validation Tasks + +# We only check the remote ssh git branch's existance if we're not running inside a container +# as getting ssh auth working inside a container seems a bit brittle +# If the main repoUpstreamURL field is set, then we need to check against +# that and not target_repo +.PHONY: validate-origin +validate-origin: ## verify the git origin is available + @echo "Checking repository:" + $(eval UPSTREAMURL := $(shell yq -r '.main.git.repoUpstreamURL // (.main.git.repoUpstreamURL = "")' values-global.yaml)) + @if [ -z "$(UPSTREAMURL)" ]; then\ + echo -n " $(TARGET_REPO) - branch '$(TARGET_BRANCH)': ";\ + git ls-remote --exit-code --heads $(TARGET_REPO) $(TARGET_BRANCH) >/dev/null &&\ + echo "OK" || (echo "NOT FOUND"; exit 1);\ + else\ + echo "Upstream URL set to: $(UPSTREAMURL)";\ + echo -n " $(UPSTREAMURL) - branch '$(TARGET_BRANCH)': ";\ + git ls-remote --exit-code --heads $(UPSTREAMURL) $(TARGET_BRANCH) >/dev/null &&\ + echo "OK" || (echo "NOT FOUND"; exit 1);\ + fi + +.PHONY: validate-cluster +validate-cluster: ## Do some cluster validations before installing + @echo "Checking cluster:" + @echo -n " cluster-info: " + @oc cluster-info >/dev/null && echo "OK" || (echo "Error"; exit 1) + @echo -n " storageclass: " + @if [ `oc get storageclass -o go-template='{{printf "%d\n" (len .items)}}'` -eq 0 ]; then\ + echo "WARNING: No storageclass found";\ + else\ + echo "OK";\ + fi + + +.PHONY: validate-schema +validate-schema: ## validates values files against schema in common/clustergroup + $(eval VAL_PARAMS := $(shell for i in ./values-*.yaml; do echo -n "$${i} "; done)) + @echo -n "Validating clustergroup schema of: " + @set -e; for i in $(VAL_PARAMS); do echo -n " $$i"; helm template oci://quay.io/hybridcloudpatterns/clustergroup $(HELM_OPTS) -f "$${i}" >/dev/null; done + @echo + +.PHONY: validate-prereq +validate-prereq: ## verify pre-requisites + $(eval GLOBAL_PATTERN := $(shell yq -r .global.pattern values-global.yaml)) + @if [ $(NAME) != $(GLOBAL_PATTERN) ]; then\ + echo "";\ + echo "WARNING: folder directory is \"$(NAME)\" and global.pattern is set to \"$(GLOBAL_PATTERN)\"";\ + echo "this can create problems. Please make sure they are the same!";\ + echo "";\ + fi + @if [ ! -f /run/.containerenv ]; then\ + echo "Checking prerequisites:";\ + echo -n " Check for python-kubernetes: ";\ + if ! ansible -m ansible.builtin.command -a "{{ ansible_python_interpreter }} -c 'import kubernetes'" localhost > /dev/null 2>&1; then echo "Not found"; exit 1; fi;\ + echo "OK";\ + echo -n " Check for kubernetes.core collection: ";\ + if ! ansible-galaxy collection list | grep kubernetes.core > /dev/null 2>&1; then echo "Not found"; exit 1; fi;\ + echo "OK";\ + else\ + if [ -f values-global.yaml ]; then\ + OUT=`yq -r '.main.multiSourceConfig.enabled // (.main.multiSourceConfig.enabled = "false")' values-global.yaml`;\ + if [ "$${OUT,,}" = "false" ]; then\ + echo "You must set \".main.multiSourceConfig.enabled: true\" in your 'values-global.yaml' file";\ + echo "because your common subfolder is the slimmed down version with no helm charts in it";\ + exit 1;\ + fi;\ + fi;\ + fi + +.PHONY: argo-healthcheck +argo-healthcheck: ## Checks if all argo applications are synced + @echo "Checking argo applications" + $(eval APPS := $(shell oc get applications.argoproj.io -A -o jsonpath='{range .items[*]}{@.metadata.namespace}{","}{@.metadata.name}{"\n"}{end}')) + @NOTOK=0; \ + for i in $(APPS); do\ + n=`echo "$${i}" | cut -f1 -d,`;\ + a=`echo "$${i}" | cut -f2 -d,`;\ + STATUS=`oc get -n "$${n}" applications.argoproj.io/"$${a}" -o jsonpath='{.status.sync.status}'`;\ + if [[ $$STATUS != "Synced" ]]; then\ + NOTOK=$$(( $${NOTOK} + 1));\ + fi;\ + HEALTH=`oc get -n "$${n}" applications.argoproj.io/"$${a}" -o jsonpath='{.status.health.status}'`;\ + if [[ $$HEALTH != "Healthy" ]]; then\ + NOTOK=$$(( $${NOTOK} + 1));\ + fi;\ + echo "$${n} $${a} -> Sync: $${STATUS} - Health: $${HEALTH}";\ + done;\ + if [ $${NOTOK} -gt 0 ]; then\ + echo "Some applications are not synced or are unhealthy";\ + exit 1;\ + fi + + +##@ Test and Linters Tasks + +.PHONY: qe-tests +qe-tests: ## Runs the tests that QE runs + @set -e; if [ -f ./tests/interop/run_tests.sh ]; then \ + pushd ./tests/interop; ./run_tests.sh; popd; \ + else \ + echo "No ./tests/interop/run_tests.sh found skipping"; \ + fi + +.PHONY: super-linter +super-linter: ## Runs super linter locally + rm -rf .mypy_cache + podman run -e RUN_LOCAL=true -e USE_FIND_ALGORITHM=true \ + -e VALIDATE_ANSIBLE=false \ + -e VALIDATE_BASH=false \ + -e VALIDATE_CHECKOV=false \ + -e VALIDATE_DOCKERFILE_HADOLINT=false \ + -e VALIDATE_JSCPD=false \ + -e VALIDATE_JSON_PRETTIER=false \ + -e VALIDATE_MARKDOWN_PRETTIER=false \ + -e VALIDATE_KUBERNETES_KUBECONFORM=false \ + -e VALIDATE_PYTHON_PYLINT=false \ + -e VALIDATE_SHELL_SHFMT=false \ + -e VALIDATE_TEKTON=false \ + -e VALIDATE_YAML=false \ + -e VALIDATE_YAML_PRETTIER=false \ + $(DISABLE_LINTERS) \ + -v $(PWD):/tmp/lint:rw,z \ + -w /tmp/lint \ + ghcr.io/super-linter/super-linter:slim-v7 + +.PHONY: deploy upgrade legacy-deploy legacy-upgrade +deploy upgrade legacy-deploy legacy-upgrade: + @echo "UNSUPPORTED TARGET: please switch to 'operator-deploy'"; exit 1 diff --git a/common/README.md b/common/README.md new file mode 100644 index 0000000..b36bc1a --- /dev/null +++ b/common/README.md @@ -0,0 +1,51 @@ +# Validated Patterns common/ repository + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +## Note + +This is the `main` branch of common and it assumes that the pattern is fully +multisource (meaning that any used charts from VP is actually referenced from +either a helm chart repository or quay repository). I.e. there are no helm +charts contained in this branch of common and there is no ansible code neither. + +The helm charts now live in separate repositories under the VP +[organization](https://github.com/validatedpatterns) on GitHub. The repositories are: + +- clustergroup-chart +- pattern-install-chart +- hashicorp-vault-chart +- golang-external-secrets-chart +- acm-chart +- letsencrypt-chart + +The ansible bits live in this [repository](https://github.com/validatedpatterns/rhvp.cluster_utils) + +In order to be able to use this "slimmed-down" main branch of common you *must* +use a 0.9.* clustergroup-chart that. Add the following to your `values-global.yaml`: + +```yaml +main: + multiSourceConfig: + enabled: true + clusterGroupChartVersion: 0.9.* +``` + +## Start Here + +This repository is never used as standalone. It is usually imported in each pattern as a subtree. +In order to import the common subtree the very first time you can use the script +[make_common_subtree.sh](scripts/make-common-subtree.sh). + +In order to update your common subtree inside your pattern repository you can either use +`https://github.com/validatedpatterns/utilities/blob/main/scripts/update-common-everywhere.sh` or +do it manually with the following commands: + +```sh +git remote add -f common-upstream https://github.com/validatedpatterns/common.git +git merge -s subtree -Xtheirs -Xsubtree=common common-upstream/main +``` + +## Secrets + +There are two different secret formats parsed by the ansible bits. Both are documented [here](https://github.com/validatedpatterns/common/tree/main/ansible/roles/vault_utils/README.md) diff --git a/common/requirements.yml b/common/requirements.yml new file mode 100644 index 0000000..cb11ca2 --- /dev/null +++ b/common/requirements.yml @@ -0,0 +1,4 @@ +--- +# Define Ansible collection requirements here +collections: + - name: git+https://github.com/validatedpatterns/rhvp.cluster_utils.git,v1 diff --git a/common/scripts/deploy-pattern.sh b/common/scripts/deploy-pattern.sh new file mode 100755 index 0000000..61074fe --- /dev/null +++ b/common/scripts/deploy-pattern.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -o pipefail + +RUNS=10 +WAIT=15 +# Retry five times because the CRD might not be fully installed yet +echo -n "Installing pattern: " +for i in $(seq 1 ${RUNS}); do \ + exec 3>&1 4>&2 + OUT=$( { helm template --include-crds --name-template $* 2>&4 | oc apply -f- 2>&4 1>&3; } 4>&1 3>&1) + ret=$? + exec 3>&- 4>&- + if [ ${ret} -eq 0 ]; then + break; + else + echo -n "." + sleep "${WAIT}" + fi +done + +# All the runs failed +if [ ${i} -eq ${RUNS} ]; then + echo "Installation failed [${i}/${RUNS}]. Error:" + echo "${OUT}" + exit 1 +fi +echo "Done" diff --git a/common/scripts/determine-main-clustergroup.sh b/common/scripts/determine-main-clustergroup.sh new file mode 100755 index 0000000..6271dba --- /dev/null +++ b/common/scripts/determine-main-clustergroup.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +PATTERN_DIR="$1" + +if [ -z "$PATTERN_DIR" ]; then + PATTERN_DIR="." +fi + +CGNAME=$(yq '.main.clusterGroupName' "$PATTERN_DIR/values-global.yaml") + +if [ -z "$CGNAME" ] || [ "$CGNAME" == "null" ]; then + echo "Error - cannot detrmine clusterGroupName" + exit 1 +fi + +echo "$CGNAME" diff --git a/common/scripts/determine-pattern-name.sh b/common/scripts/determine-pattern-name.sh new file mode 100755 index 0000000..fb503fe --- /dev/null +++ b/common/scripts/determine-pattern-name.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +PATTERN_DIR="$1" + +if [ -z "$PATTERN_DIR" ]; then + PATTERN_DIR="." +fi + +PATNAME=$(yq '.global.pattern' "$PATTERN_DIR/values-global.yaml" 2>/dev/null) + +if [ -z "$PATNAME" ] || [ "$PATNAME" == "null" ]; then + PATNAME="$(basename "$PWD")" +fi + +echo "$PATNAME" diff --git a/common/scripts/determine-secretstore-backend.sh b/common/scripts/determine-secretstore-backend.sh new file mode 100755 index 0000000..ef78479 --- /dev/null +++ b/common/scripts/determine-secretstore-backend.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +PATTERN_DIR="$1" + +if [ -z "$PATTERN_DIR" ]; then + PATTERN_DIR="." +fi + +BACKEND=$(yq '.global.secretStore.backend' "$PATTERN_DIR/values-global.yaml" 2>/dev/null) + +if [ -z "$BACKEND" -o "$BACKEND" == "null" ]; then + BACKEND="vault" +fi + +echo "$BACKEND" diff --git a/common/scripts/display-secrets-info.sh b/common/scripts/display-secrets-info.sh new file mode 100755 index 0000000..ca0069e --- /dev/null +++ b/common/scripts/display-secrets-info.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -eu + +get_abs_filename() { + # $1 : relative filename + echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +} + +SCRIPT=$(get_abs_filename "$0") +SCRIPTPATH=$(dirname "${SCRIPT}") +COMMONPATH=$(dirname "${SCRIPTPATH}") +PATTERNPATH=$(dirname "${COMMONPATH}") + +if [ "$#" -ge 1 ]; then + export VALUES_SECRET=$(get_abs_filename "${1}") +fi + +if [[ "$#" == 2 ]]; then + SECRETS_BACKING_STORE="$2" +else + SECRETS_BACKING_STORE="$($SCRIPTPATH/determine-secretstore-backend.sh)" +fi + +PATTERN_NAME=$(basename "`pwd`") + +EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}" + +ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" -e secrets_backing_store="${SECRETS_BACKING_STORE}" -e hide_sensitive_output=false ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.display_secrets_info" diff --git a/common/scripts/load-k8s-secrets.sh b/common/scripts/load-k8s-secrets.sh new file mode 100755 index 0000000..707e51a --- /dev/null +++ b/common/scripts/load-k8s-secrets.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -eu + +get_abs_filename() { + # $1 : relative filename + echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +} + +SCRIPT=$(get_abs_filename "$0") +SCRIPTPATH=$(dirname "${SCRIPT}") +COMMONPATH=$(dirname "${SCRIPTPATH}") +PATTERNPATH=$(dirname "${COMMONPATH}") + +PATTERN_NAME=${1:-$(basename "`pwd`")} + +EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}" + +ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.k8s_secrets" diff --git a/common/scripts/make-common-subtree.sh b/common/scripts/make-common-subtree.sh new file mode 100755 index 0000000..196a4c8 --- /dev/null +++ b/common/scripts/make-common-subtree.sh @@ -0,0 +1,76 @@ +#!/bin/sh + +if [ "$1" = "-h" ]; then + echo "This script will convert common into a subtree and add a remote to help manage it." + echo "The script takes three positional arguments, as follows:" + echo + echo "$0 " + echo + echo "Run without arguments, the script would run as if these arguments had been passed:" + echo "$0 https://github.com/validatedpatterns/common.git main common-upstream" + echo + echo "Please ensure the git subtree command is available. On RHEL/Fedora, the git subtree command" + echo "is in a separate package called git-subtree" + exit 1 +fi + +if [ -f '/etc/redhat-release' ]; then + rpm -qa | grep git-subtree 2>&1 + if [ ! $? = 0 ]; then + echo "you need to install git-subtree" + echo "would you like to install it now?" + select ANS in yes no + do + case $ANS in + yes) + sudo dnf install git-subtree -y + break + ;; + no) + exit + break + ;; + *) + echo "You must enter yes or no" + ;; + esac + done + fi +fi + +if [ "$1" ]; then + subtree_repo=$1 +else + subtree_repo=https://github.com/validatedpatterns/common.git +fi + +if [ "$2" ]; then + subtree_branch=$2 +else + subtree_branch=main +fi + +if [ "$3" ]; then + subtree_remote=$3 +else + subtree_remote=common-upstream +fi + +git diff --quiet || (echo "This script must be run on a clean working tree" && exit 1) + +echo "Changing directory to project root" +cd `git rev-parse --show-toplevel` + +echo "Removing existing common and replacing it with subtree from $subtree_repo $subtree_remote" +rm -rf common + +echo "Committing removal of common" +(git add -A :/ && git commit -m "Removed previous version of common to convert to subtree from $subtree_repo $subtree_branch") || exit 1 + +echo "Adding (possibly replacing) subtree remote $subtree_remote" +git remote rm "$subtree_remote" +git remote add -f "$subtree_remote" "$subtree_repo" || exit 1 +git subtree add --prefix=common "$subtree_remote" "$subtree_branch" || exit 1 + +echo "Complete. You may now push these results if you are satisfied" +exit 0 diff --git a/common/scripts/manage-secret-app.sh b/common/scripts/manage-secret-app.sh new file mode 100755 index 0000000..18a986e --- /dev/null +++ b/common/scripts/manage-secret-app.sh @@ -0,0 +1,53 @@ +#!/bin/sh + +APP=$1 +STATE=$2 + +MAIN_CLUSTERGROUP_FILE="./values-$(common/scripts/determine-main-clustergroup.sh).yaml" +MAIN_CLUSTERGROUP_PROJECT="$(common/scripts/determine-main-clustergroup.sh)" + +case "$APP" in + "vault") + APP_NAME="vault" + NAMESPACE="vault" + PROJECT="$MAIN_CLUSTERGROUP_PROJECT" + CHART_NAME="hashicorp-vault" + CHART_VERSION=0.1.* + + ;; + "golang-external-secrets") + APP_NAME="golang-external-secrets" + NAMESPACE="golang-external-secrets" + PROJECT="$MAIN_CLUSTERGROUP_PROJECT" + CHART_NAME="golang-external-secrets" + CHART_VERSION=0.1.* + + ;; + *) + echo "Error - cannot manage $APP can only manage vault and golang-external-secrets" + exit 1 + ;; +esac + +case "$STATE" in + "present") + common/scripts/manage-secret-namespace.sh "$NAMESPACE" "$STATE" + + RES=$(yq ".clusterGroup.applications[] | select(.path == \"$CHART_LOCATION\")" "$MAIN_CLUSTERGROUP_FILE" 2>/dev/null) + if [ -z "$RES" ]; then + echo "Application with chart location $CHART_LOCATION not found, adding" + yq -i ".clusterGroup.applications.$APP_NAME = { \"name\": \"$APP_NAME\", \"namespace\": \"$NAMESPACE\", \"project\": \"$PROJECT\", \"chart\": \"$CHART_NAME\", \"chartVersion\": \"$CHART_VERSION\"}" "$MAIN_CLUSTERGROUP_FILE" + fi + ;; + "absent") + common/scripts/manage-secret-namespace.sh "$NAMESPACE" "$STATE" + echo "Removing application wth chart location $CHART_LOCATION" + yq -i "del(.clusterGroup.applications[] | select(.chart == \"$CHART_NAME\"))" "$MAIN_CLUSTERGROUP_FILE" + ;; + *) + echo "$STATE not supported" + exit 1 + ;; +esac + +exit 0 diff --git a/common/scripts/manage-secret-namespace.sh b/common/scripts/manage-secret-namespace.sh new file mode 100755 index 0000000..bcb0674 --- /dev/null +++ b/common/scripts/manage-secret-namespace.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +NAMESPACE=$1 +STATE=$2 + +MAIN_CLUSTERGROUP_FILE="./values-$(common/scripts/determine-main-clustergroup.sh).yaml" +MAIN_CLUSTERGROUP_PROJECT="$(common/scripts/determine-main-clustergroup.sh)" + +case "$STATE" in + "present") + + RES=$(yq ".clusterGroup.namespaces[] | select(. == \"$NAMESPACE\")" "$MAIN_CLUSTERGROUP_FILE" 2>/dev/null) + if [ -z "$RES" ]; then + echo "Namespace $NAMESPACE not found, adding" + yq -i ".clusterGroup.namespaces += [ \"$NAMESPACE\" ]" "$MAIN_CLUSTERGROUP_FILE" + fi + ;; + "absent") + echo "Removing namespace $NAMESPACE" + yq -i "del(.clusterGroup.namespaces[] | select(. == \"$NAMESPACE\"))" "$MAIN_CLUSTERGROUP_FILE" + ;; + *) + echo "$STATE not supported" + exit 1 + ;; +esac + +exit 0 diff --git a/common/scripts/pattern-util.sh b/common/scripts/pattern-util.sh new file mode 100755 index 0000000..8258d46 --- /dev/null +++ b/common/scripts/pattern-util.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +function is_available { + command -v $1 >/dev/null 2>&1 || { echo >&2 "$1 is required but it's not installed. Aborting."; exit 1; } +} + +function version { + echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }' +} + +if [ -z "$PATTERN_UTILITY_CONTAINER" ]; then + PATTERN_UTILITY_CONTAINER="quay.io/hybridcloudpatterns/utility-container" +fi +# If PATTERN_DISCONNECTED_HOME is set it will be used to populate both PATTERN_UTILITY_CONTAINER +# and PATTERN_INSTALL_CHART automatically +if [ -n "${PATTERN_DISCONNECTED_HOME}" ]; then + PATTERN_UTILITY_CONTAINER="${PATTERN_DISCONNECTED_HOME}/utility-container" + PATTERN_INSTALL_CHART="oci://${PATTERN_DISCONNECTED_HOME}/pattern-install" + echo "PATTERN_DISCONNECTED_HOME is set to ${PATTERN_DISCONNECTED_HOME}" + echo "Setting the following variables:" + echo " PATTERN_UTILITY_CONTAINER: ${PATTERN_UTILITY_CONTAINER}" + echo " PATTERN_INSTALL_CHART: ${PATTERN_INSTALL_CHART}" +fi + +readonly commands=(podman) +for cmd in ${commands[@]}; do is_available "$cmd"; done + +UNSUPPORTED_PODMAN_VERSIONS="1.6 1.5" +PODMAN_VERSION_STR=$(podman --version) +for i in ${UNSUPPORTED_PODMAN_VERSIONS}; do + # We add a space + if echo "${PODMAN_VERSION_STR}" | grep -q -E "\b${i}"; then + echo "Unsupported podman version. We recommend > 4.3.0" + podman --version + exit 1 + fi +done + +# podman --version outputs: +# podman version 4.8.2 +PODMAN_VERSION=$(echo "${PODMAN_VERSION_STR}" | awk '{ print $NF }') + +# podman < 4.3.0 do not support keep-id:uid=... +if [ $(version "${PODMAN_VERSION}") -lt $(version "4.3.0") ]; then + PODMAN_ARGS="-v ${HOME}:/root" +else + # We do not rely on bash's $UID and $GID because on MacOSX $GID is not set + MYNAME=$(id -n -u) + MYUID=$(id -u) + MYGID=$(id -g) + PODMAN_ARGS="--passwd-entry ${MYNAME}:x:${MYUID}:${MYGID}::/pattern-home:/bin/bash --user ${MYUID}:${MYGID} --userns keep-id:uid=${MYUID},gid=${MYGID}" + +fi + +if [ -n "$KUBECONFIG" ]; then + if [[ ! "${KUBECONFIG}" =~ ^$HOME* ]]; then + echo "${KUBECONFIG} is pointing outside of the HOME folder, this will make it unavailable from the container." + echo "Please move it somewhere inside your $HOME folder, as that is what gets bind-mounted inside the container" + exit 1 + fi +fi + +# Detect if we use podman machine. If we do not then we bind mount local host ssl folders +# if we are using podman machine then we do not bind mount anything (for now!) +REMOTE_PODMAN=$(podman system connection list -q | wc -l) +if [ $REMOTE_PODMAN -eq 0 ]; then # If we are not using podman machine we check the hosts folders + # We check /etc/pki/tls because on ubuntu /etc/pki/fwupd sometimes + # exists but not /etc/pki/tls and we do not want to bind mount in such a case + # as it would find no certificates at all. + if [ -d /etc/pki/tls ]; then + PKI_HOST_MOUNT_ARGS="-v /etc/pki:/etc/pki:ro" + elif [ -d /etc/ssl ]; then + PKI_HOST_MOUNT_ARGS="-v /etc/ssl:/etc/ssl:ro" + else + PKI_HOST_MOUNT_ARGS="-v /usr/share/ca-certificates:/usr/share/ca-certificates:ro" + fi +else + PKI_HOST_MOUNT_ARGS="" +fi + +# Copy Kubeconfig from current environment. The utilities will pick up ~/.kube/config if set so it's not mandatory +# $HOME is mounted as itself for any files that are referenced with absolute paths +# $HOME is mounted to /root because the UID in the container is 0 and that's where SSH looks for credentials + +podman run -it --rm --pull=newer \ + --security-opt label=disable \ + -e EXTRA_HELM_OPTS \ + -e EXTRA_PLAYBOOK_OPTS \ + -e TARGET_ORIGIN \ + -e TARGET_SITE \ + -e TARGET_BRANCH \ + -e NAME \ + -e TOKEN_SECRET \ + -e TOKEN_NAMESPACE \ + -e VALUES_SECRET \ + -e KUBECONFIG \ + -e PATTERN_INSTALL_CHART \ + -e PATTERN_DISCONNECTED_HOME \ + -e K8S_AUTH_HOST \ + -e K8S_AUTH_VERIFY_SSL \ + -e K8S_AUTH_SSL_CA_CERT \ + -e K8S_AUTH_USERNAME \ + -e K8S_AUTH_PASSWORD \ + -e K8S_AUTH_TOKEN \ + ${PKI_HOST_MOUNT_ARGS} \ + -v "${HOME}":"${HOME}" \ + -v "${HOME}":/pattern-home \ + ${PODMAN_ARGS} \ + ${EXTRA_ARGS} \ + -w "$(pwd)" \ + "$PATTERN_UTILITY_CONTAINER" \ + $@ diff --git a/common/scripts/preview-all.sh b/common/scripts/preview-all.sh new file mode 100755 index 0000000..4bf5932 --- /dev/null +++ b/common/scripts/preview-all.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +REPO=$1; shift; +TARGET_BRANCH=$1; shift + +HUB=$( yq ".main.clusterGroupName" values-global.yaml ) +MANAGED_CLUSTERS=$( yq ".clusterGroup.managedClusterGroups.[].name" values-$HUB.yaml ) +ALL_CLUSTERS=( $HUB $MANAGED_CLUSTERS ) + +CLUSTER_INFO_OUT=$(oc cluster-info 2>&1) +CLUSTER_INFO_RET=$? +if [ $CLUSTER_INFO_RET -ne 0 ]; then + echo "Could not access the cluster:" + echo "${CLUSTER_INFO_OUT}" + exit 1 +fi + +for cluster in ${ALL_CLUSTERS[@]}; do + # We always add clustergroup as it is the entry point and it gets special cased in preview.sh. + APPS="clustergroup $( yq ".clusterGroup.applications.[].name" values-$cluster.yaml )" + for app in $APPS; do + printf "# Parsing application $app from cluster $cluster\n" + common/scripts/preview.sh $cluster $app $REPO $TARGET_BRANCH + done +done diff --git a/common/scripts/preview.sh b/common/scripts/preview.sh new file mode 100755 index 0000000..6da4578 --- /dev/null +++ b/common/scripts/preview.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# DISCLAIMER +# +# - Parsing of applications needs to be more clever. +# - There is currently not a mechanism to actually preview against multiple clusters +# (i.e. a hub and a remote). All previews will be done against the current. +# - Make output can be included in the YAML. + +SITE=$1; shift +APPNAME=$1; shift +GIT_REPO=$1; shift +GIT_BRANCH=$1; shift + +if [ "${APPNAME}" != "clustergroup" ]; then + # This covers the following case: + # foobar: + # name: foo + # namespace: foo + # project: foo + # path: charts/all/foo + # So we retrieve the actual index ("foobar") given the name attribute of the application + APP=$(yq ".clusterGroup.applications | with_entries(select(.value.name == \"$APPNAME\")) | keys | .[0]" values-$SITE.yaml) + isLocalHelmChart=$(yq ".clusterGroup.applications.$APP.path" values-$SITE.yaml) + if [ $isLocalHelmChart != "null" ]; then + chart=$(yq ".clusterGroup.applications.$APP.path" values-$SITE.yaml) + else + helmrepo=$(yq ".clusterGroup.applications.$APP.repoURL" values-$SITE.yaml) + helmrepo="${helmrepo:+oci://quay.io/hybridcloudpatterns}" + chartversion=$(yq ".clusterGroup.applications.$APP.chartVersion" values-$SITE.yaml) + chartname=$(yq ".clusterGroup.applications.$APP.chart" values-$SITE.yaml) + chart="${helmrepo}/${chartname} --version ${chartversion}" + fi + namespace=$(yq ".clusterGroup.applications.$APP.namespace" values-$SITE.yaml) +else + APP=$APPNAME + clusterGroupChartVersion=$(yq ".main.multiSourceConfig.clusterGroupChartVersion" values-global.yaml) + helmrepo="oci://quay.io/hybridcloudpatterns" + chart="${helmrepo}/clustergroup --version ${clusterGroupChartVersion}" + namespace="openshift-operators" +fi +pattern=$(yq ".global.pattern" values-global.yaml) + +# You can override the default lookups by using OCP_{PLATFORM,VERSION,DOMAIN} +# Note that when using the utility container you need to pass in the above variables +# by export EXTRA_ARGS="-e OCP_PLATFORM -e OCP_VERSION -e OCP_DOMAIN" before +# invoking pattern-util.sh +platform=${OCP_PLATFORM:-$(oc get Infrastructure.config.openshift.io/cluster -o jsonpath='{.spec.platformSpec.type}')} +ocpversion=${OCP_VERSION:-$(oc get clusterversion/version -o jsonpath='{.status.desired.version}' | awk -F. '{print $1"."$2}')} +domain=${OCP_DOMAIN:-$(oc get Ingress.config.openshift.io/cluster -o jsonpath='{.spec.domain}' | sed 's/^apps.//')} + +function replaceGlobals() { + output=$( echo $1 | sed -e 's/ //g' -e 's/\$//g' -e s@^-@@g -e s@\'@@g ) + + output=$(echo $output | sed "s@{{.Values.global.clusterPlatform}}@${platform}@g") + output=$(echo $output | sed "s@{{.Values.global.clusterVersion}}@${ocpversion}@g") + output=$(echo $output | sed "s@{{.Values.global.clusterDomain}}@${domain}@g") + + echo $output +} + +function getOverrides() { + overrides='' + overrides=$( yq ".clusterGroup.applications.$APP.overrides[]" "values-$SITE.yaml" ) + overrides=$( echo "$overrides" | tr -d '\n' ) + overrides=$( echo "$overrides" | sed -e 's/name:/ --set/g; s/value: /=/g' ) + if [ -n "$overrides" ]; then + echo "$overrides" + fi +} + + +CLUSTER_OPTS="" +CLUSTER_OPTS="$CLUSTER_OPTS --set global.pattern=$pattern" +CLUSTER_OPTS="$CLUSTER_OPTS --set global.repoURL=$GIT_REPO" +CLUSTER_OPTS="$CLUSTER_OPTS --set main.git.repoURL=$GIT_REPO" +CLUSTER_OPTS="$CLUSTER_OPTS --set main.git.revision=$GIT_BRANCH" +CLUSTER_OPTS="$CLUSTER_OPTS --set global.namespace=$namespace" +CLUSTER_OPTS="$CLUSTER_OPTS --set global.hubClusterDomain=apps.$domain" +CLUSTER_OPTS="$CLUSTER_OPTS --set global.localClusterDomain=apps.$domain" +CLUSTER_OPTS="$CLUSTER_OPTS --set global.clusterDomain=$domain" +CLUSTER_OPTS="$CLUSTER_OPTS --set global.clusterVersion=$ocpversion" +CLUSTER_OPTS="$CLUSTER_OPTS --set global.clusterPlatform=$platform" + + +sharedValueFiles=$(yq ".clusterGroup.sharedValueFiles" values-$SITE.yaml) +appValueFiles=$(yq ".clusterGroup.applications.$APP.extraValueFiles" values-$SITE.yaml) +isKustomize=$(yq ".clusterGroup.applications.$APP.kustomize" values-$SITE.yaml) +OVERRIDES=$( getOverrides ) + +VALUE_FILES="-f values-global.yaml -f values-$SITE.yaml" +IFS=$'\n' +for line in $sharedValueFiles; do + if [ $line != "null" ] && [ -f $line ]; then + file=$(replaceGlobals $line) + VALUE_FILES="$VALUE_FILES -f $PWD$file" + fi +done + +for line in $appValueFiles; do + if [ $line != "null" ] && [ -f $line ]; then + file=$(replaceGlobals $line) + VALUE_FILES="$VALUE_FILES -f $PWD$file" + fi +done + +if [ $isKustomize == "true" ]; then + kustomizePath=$(yq ".clusterGroup.applications.$APP.path" values-$SITE.yaml) + repoURL=$(yq ".clusterGroup.applications.$APP.repoURL" values-$SITE.yaml) + if [[ $repoURL == http* ]] || [[ $repoURL == git@ ]]; then + kustomizePath="${repoURL}/${kustomizePath}" + fi + cmd="oc kustomize ${kustomizePath}" + eval "$cmd" +else + cmd="helm template $chart --name-template ${APP} -n ${namespace} ${VALUE_FILES} ${OVERRIDES} ${CLUSTER_OPTS}" + eval "$cmd" +fi diff --git a/common/scripts/process-secrets.sh b/common/scripts/process-secrets.sh new file mode 100755 index 0000000..a0d34f8 --- /dev/null +++ b/common/scripts/process-secrets.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -eu + +get_abs_filename() { + # $1 : relative filename + echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +} + +SCRIPT=$(get_abs_filename "$0") +SCRIPTPATH=$(dirname "${SCRIPT}") +COMMONPATH=$(dirname "${SCRIPTPATH}") +PATTERNPATH=$(dirname "${COMMONPATH}") + +PATTERN_NAME=${1:-$(basename "`pwd`")} +SECRETS_BACKING_STORE="$($SCRIPTPATH/determine-secretstore-backend.sh)" + +EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}" + +ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" -e secrets_backing_store="${SECRETS_BACKING_STORE}" ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.process_secrets" diff --git a/common/scripts/set-secret-backend.sh b/common/scripts/set-secret-backend.sh new file mode 100755 index 0000000..e07b15b --- /dev/null +++ b/common/scripts/set-secret-backend.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +BACKEND=$1 + +yq -i ".global.secretStore.backend = \"$BACKEND\"" values-global.yaml diff --git a/common/scripts/vault-utils.sh b/common/scripts/vault-utils.sh new file mode 100755 index 0000000..2f76649 --- /dev/null +++ b/common/scripts/vault-utils.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -eu + +get_abs_filename() { + # $1 : relative filename + echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +} + +SCRIPT=$(get_abs_filename "$0") +SCRIPTPATH=$(dirname "${SCRIPT}") +COMMONPATH=$(dirname "${SCRIPTPATH}") +PATTERNPATH=$(dirname "${COMMONPATH}") + +# Parse arguments +if [ $# -lt 1 ]; then + echo "Specify at least the command ($#): $*" + exit 1 +fi + +TASK="${1}" +PATTERN_NAME=${2:-$(basename "`pwd`")} + +if [ -z ${TASK} ]; then + echo "Task is unset" + exit 1 +fi + +EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}" + +ansible-playbook -t "${TASK}" -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.vault" diff --git a/common/scripts/write-token-kubeconfig.sh b/common/scripts/write-token-kubeconfig.sh new file mode 100755 index 0000000..e7913e5 --- /dev/null +++ b/common/scripts/write-token-kubeconfig.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -eu + +OUTPUTFILE=${1:-"~/.kube/config"} + +get_abs_filename() { + # $1 : relative filename + echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +} + +SCRIPT=$(get_abs_filename "$0") +SCRIPTPATH=$(dirname "${SCRIPT}") +COMMONPATH=$(dirname "${SCRIPTPATH}") +PATTERNPATH=$(dirname "${COMMONPATH}") + +EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}" + +ansible-playbook -e pattern_dir="${PATTERNPATH}" -e kubeconfig_file="${OUTPUTFILE}" ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.write-token-kubeconfig" diff --git a/overrides/values-AWS.yaml b/overrides/values-AWS.yaml new file mode 100644 index 0000000..03fa077 --- /dev/null +++ b/overrides/values-AWS.yaml @@ -0,0 +1,26 @@ +# The following snippet can be commented out in oroder +# to enable letsencrypt certificates on API endpoint and default +# ingress of the cluster +# It is currently very experimental and unsupported. +# PLEASE read https://github.com/hybrid-cloud-patterns/common/tree/main/letsencrypt#readme +# for all the limitations around it + + +# letsencrypt: +# enabled: true +# api_endpoint: true +# # FIXME: tweak this to match your region +# region: eu-central-1 +# server: https://acme-v02.api.letsencrypt.org/directory +# # server: https://acme-staging-v02.api.letsencrypt.org/directory +# # FIXME: set this to your correct email +# email: iwashere@iwashere.com +# +# clusterGroup: +# applications: +# letsencrypt: +# name: letsencrypt +# namespace: letsencrypt +# # Using 'default' as that exists everywhere +# project: default +# path: common/letsencrypt diff --git a/overrides/values-IBMCloud.yaml b/overrides/values-IBMCloud.yaml new file mode 100644 index 0000000..38d7be7 --- /dev/null +++ b/overrides/values-IBMCloud.yaml @@ -0,0 +1,10 @@ +# When using IBM ROKS the route certificates are signed by letsencrypt +# By default the ESO configuration uses the kube-root-ca.crt configmap +# to validate the connection to vault. Since this configmap will not contain +# the letsencrypt CA, ESO will be unable to connect to the vault and return an +# x509 CA unknown error. +# Uncomment the following if you are using IBM ROKS (IPI installs on IBM Cloud are unaffected) + +# golangExternalSecrets: +# caProvider: +# enabled: false diff --git a/pattern-metadata.yaml b/pattern-metadata.yaml new file mode 100644 index 0000000..fdb3a0d --- /dev/null +++ b/pattern-metadata.yaml @@ -0,0 +1,70 @@ +# This goal of this metadata is mainly used as a source of truth for +# documentation and qe +metadata_version: "1.0" +name: multicloud-gitops +pattern_version: "1.0" +display_name: Multicloud Gitops +repo_url: https://github.com/validatedpatterns/multicloud-gitops +docs_repo_url: https://github.com/validatedpatterns/docs +issues_url: https://github.com/validatedpatterns/multicloud-gitops/issues +docs_url: https://validatedpatterns.io/patterns/multicloud-gitops/ +ci_url: https://validatedpatterns.io/ci/?pattern=mcgitops +# can be sandbox, tested or maintained +tier: maintained +owners: mbaldessari, darkdoc +requirements: + hub: # Main cluster + compute: + platform: + gcp: + replicas: 3 + type: n1-standard-8 + azure: + replicas: 3 + type: Standard_D8s_v3 + aws: + replicas: 3 + type: m5.2xlarge + controlPlane: + platform: + gcp: + replicas: 3 + type: n1-standard-4 + azure: + replicas: 3 + type: Standard_D4s_v3 + aws: + replicas: 3 + type: m5.xlarge + spoke: # optional - represents the clusters imported into ACM + compute: + platform: + gcp: + replicas: 0 + type: n1-standard-8 + azure: + replicas: 0 + type: Standard_D8s_v3 + aws: + replicas: 0 + type: m5.2xlarge + controlPlane: + platform: + gcp: + replicas: 3 + type: n1-standard-8 + azure: + replicas: 3 + type: Standard_D8s_v3 + aws: + replicas: 3 + type: m5.2xlarge + +# Loosely defined extra features like hypershift support, non-openshift +# kubernetes support, spoke support +extra_features: + hypershift_support: true + spoke_support: true + +external_requirements: +# external quay, s3 bucket, agof tokens to access paywalled material, manifests, rag-llm hw (only selected regions) diff --git a/pattern.sh b/pattern.sh new file mode 120000 index 0000000..223a550 --- /dev/null +++ b/pattern.sh @@ -0,0 +1 @@ +./common/scripts/pattern-util.sh \ No newline at end of file diff --git a/tests/interop/README.md b/tests/interop/README.md new file mode 100644 index 0000000..54560a1 --- /dev/null +++ b/tests/interop/README.md @@ -0,0 +1,25 @@ +# Running tests + +## Prerequisites + +* Openshift clusters with multicloud-gitops pattern installed + * factory cluster is managed via rhacm +* kubeconfig files for Openshift clusters +* oc client installed at ~/oc_client/oc + +## Steps + +* create python3 venv, clone multicloud-gitops repository +* export KUBECONFIG=\ +* export KUBECONFIG_EDGE=\ +* export INFRA_PROVIDER=\ +* (optional) export WORKSPACE=\ (defaults to /tmp) +* cd multicloud-gitops/tests/interop +* pip install -r requirements.txt +* ./run_tests.sh + +## Results + +* results .xml files will be placed at $WORKSPACE +* test logs will be placed at $WORKSPACE/.results/test_execution_logs/ +* CI badge file will be placed at $WORKSPACE diff --git a/tests/interop/__init__.py b/tests/interop/__init__.py new file mode 100644 index 0000000..890362c --- /dev/null +++ b/tests/interop/__init__.py @@ -0,0 +1,2 @@ +__version__ = "0.1.0" +__loggername__ = "css_logger" diff --git a/tests/interop/conftest.py b/tests/interop/conftest.py new file mode 100644 index 0000000..fb301d5 --- /dev/null +++ b/tests/interop/conftest.py @@ -0,0 +1,2 @@ +from validatedpatterns_tests.interop.conftest_logger import * # noqa: F401, F403 +from validatedpatterns_tests.interop.conftest_openshift import * # noqa: F401, F403 diff --git a/tests/interop/create_ci_badge.py b/tests/interop/create_ci_badge.py new file mode 100644 index 0000000..8ed179a --- /dev/null +++ b/tests/interop/create_ci_badge.py @@ -0,0 +1,84 @@ +import json +import os +import subprocess +from datetime import datetime + +from junitparser import JUnitXml + +oc = os.environ["HOME"] + "/oc_client/oc" + +ci_badge = { + "schemaVersion": 1, + "label": "Community test", + "message": "", + "color": "red", + "openshiftVersion": "", + "infraProvider": os.environ.get("INFRA_PROVIDER"), + "patternName": os.environ.get("PATTERN_NAME"), + "patternRepo": "", + "patternBranch": "", + "date": datetime.today().strftime("%Y-%m-%d"), + "testSource": "Community", + "debugInfo": None, +} + + +def get_openshift_version(): + try: + version_ret = subprocess.run([oc, "version", "-o", "json"], capture_output=True) + version_out = version_ret.stdout.decode("utf-8") + openshift_version = json.loads(version_out)["openshiftVersion"] + major_minor = ".".join(openshift_version.split(".")[:-1]) + return openshift_version, major_minor + except KeyError as e: + print("KeyError:" + str(e)) + return None + + +if __name__ == "__main__": + versions = get_openshift_version() + ci_badge["openshiftVersion"] = versions[0] + + pattern_repo = subprocess.run( + ["git", "config", "--get", "remote.origin.url"], capture_output=True, text=True + ) + pattern_branch = subprocess.run( + ["git", "branch", "--show-current"], capture_output=True, text=True + ) + + ci_badge["patternRepo"] = pattern_repo.stdout.strip() + ci_badge["patternBranch"] = pattern_branch.stdout.strip() + + # Check each xml file for failures + results_dir = os.environ.get("WORKSPACE") + failures = 0 + + for file in os.listdir(results_dir): + if file.startswith("test_") and file.endswith(".xml"): + with open(os.path.join(results_dir, file), "r") as result_file: # type: ignore + xml = JUnitXml.fromfile(result_file) # type: ignore + for suite in xml: + for case in suite: + if case.result: + failures += 1 + + # Determine badge color from results + if failures == 0: + ci_badge["color"] = "green" + + # For now we assume `message` is the same as patternBranch + ci_badge["message"] = ci_badge["patternBranch"] + + ci_badge_json_basename = ( + os.environ.get("PATTERN_SHORTNAME") # type: ignore + + "-" + + os.environ.get("INFRA_PROVIDER") + + "-" + + versions[1] + + "-stable-badge.json" + ) + ci_badge_json_filename = os.path.join(results_dir, ci_badge_json_basename) # type: ignore + print(f"Creating CI badge file at: {ci_badge_json_filename}") + + with open(ci_badge_json_filename, "w") as ci_badge_file: + json.dump(ci_badge, ci_badge_file) diff --git a/tests/interop/requirements.txt b/tests/interop/requirements.txt new file mode 100644 index 0000000..3b20852 --- /dev/null +++ b/tests/interop/requirements.txt @@ -0,0 +1,6 @@ +pytest +kubernetes +openshift +openshift-python-wrapper +junitparser +git+https://github.com/validatedpatterns/vp-qe-test-common.git@development#egg=vp-qe-test-common \ No newline at end of file diff --git a/tests/interop/run_tests.sh b/tests/interop/run_tests.sh new file mode 100755 index 0000000..a1af3b4 --- /dev/null +++ b/tests/interop/run_tests.sh @@ -0,0 +1,36 @@ +#!/usr/bin/bash + +export EXTERNAL_TEST="true" +export PATTERN_NAME="MultiCloudGitops" +export PATTERN_SHORTNAME="mcgitops" + +if [ -z "${KUBECONFIG}" ]; then + echo "No kubeconfig file set for hub cluster" + exit 1 +fi + +if [ -z "${KUBECONFIG_EDGE}" ]; then + echo "No kubeconfig file set for edge cluster" + exit 1 +fi + +if [ -z "${INFRA_PROVIDER}" ]; then + echo "INFRA_PROVIDER is not defined" + exit 1 +fi + +if [ -z "${WORKSPACE}" ]; then + export WORKSPACE=/tmp +fi + +pytest -lv --disable-warnings test_subscription_status_hub.py --kubeconfig $KUBECONFIG --junit-xml $WORKSPACE/test_subscription_status_hub.xml + +pytest -lv --disable-warnings test_subscription_status_edge.py --kubeconfig $KUBECONFIG_EDGE --junit-xml $WORKSPACE/test_subscription_status_edge.xml + +pytest -lv --disable-warnings test_validate_hub_site_components.py --kubeconfig $KUBECONFIG --junit-xml $WORKSPACE/test_validate_hub_site_components.xml + +pytest -lv --disable-warnings test_validate_edge_site_components.py --kubeconfig $KUBECONFIG_EDGE --junit-xml $WORKSPACE/test_validate_edge_site_components.xml + +pytest -lv --disable-warnings test_modify_web_content.py --kubeconfig $KUBECONFIG --junit-xml $WORKSPACE/test_modify_web_content.xml + +python3 create_ci_badge.py diff --git a/tests/interop/test_modify_web_content.py b/tests/interop/test_modify_web_content.py new file mode 100644 index 0000000..44cf045 --- /dev/null +++ b/tests/interop/test_modify_web_content.py @@ -0,0 +1,89 @@ +import logging +import os +import re +import subprocess +import time + +import pytest +import requests +from ocp_resources.route import Route +from openshift.dynamic.exceptions import NotFoundError +from validatedpatterns_tests.interop.edge_util import modify_file_content + +from . import __loggername__ + +logger = logging.getLogger(__loggername__) + + +@pytest.mark.modify_web_content +def test_modify_web_content(openshift_dyn_client): + logger.info("Find the url for the hello-world route") + try: + for route in Route.get( + dyn_client=openshift_dyn_client, + namespace="hello-world", + name="hello-world", + ): + logger.info(route.instance.spec.host) + except NotFoundError: + err_msg = "hello-world url/route is missing in hello-world namespace" + logger.error(f"FAIL: {err_msg}") + assert False, err_msg + + url = "http://" + route.instance.spec.host + response = requests.get(url) + logger.info(f"Current page content: {response.content}") + + if os.getenv("EXTERNAL_TEST") != "true": + chart = ( + f"{os.environ['HOME']}" + + "/validated_patterns/multicloud-gitops/charts/" + + "all/hello-world/templates/hello-world-cm.yaml" + ) + else: + chart = "../../charts/all/hello-world/templates/hello-world-cm.yaml" + + logger.info("Modify the file content") + orig_heading = "

Hello World!

" + new_heading = "

Validated Patterns QE was here!

" + modify_file_content( + file_name=chart, orig_content=orig_heading, new_content=new_heading + ) + + logger.info("Merge the change") + patterns_repo = f"{os.environ['HOME']}/validated_patterns/multicloud-gitops" + if os.getenv("EXTERNAL_TEST") != "true": + subprocess.run(["git", "add", chart], cwd=f"{patterns_repo}") + subprocess.run( + ["git", "commit", "-m", "Updating 'hello-world'"], cwd=f"{patterns_repo}" + ) + push = subprocess.run( + ["git", "push"], cwd=f"{patterns_repo}", capture_output=True, text=True + ) + else: + subprocess.run(["git", "add", chart]) + subprocess.run(["git", "commit", "-m", "Updating 'hello-world'"]) + push = subprocess.run(["git", "push"], capture_output=True, text=True) + logger.info(push.stdout) + logger.info(push.stderr) + + logger.info("Checking for updated page content") + timeout = time.time() + 60 * 10 + while time.time() < timeout: + time.sleep(30) + response = requests.get(url) + logger.info(response.content) + + new_content = re.search(new_heading, str(response.content)) + + logger.info(new_content) + if (new_content is None) or (new_content.group() != new_heading): + continue + break + + if (new_content is None) or (new_content.group() != new_heading): + err_msg = "Did not find updated page content" + logger.error(f"FAIL: {err_msg}") + assert False, err_msg + else: + logger.info("PASS: Found updated page content") diff --git a/tests/interop/test_subscription_status_edge.py b/tests/interop/test_subscription_status_edge.py new file mode 100644 index 0000000..d8ef34a --- /dev/null +++ b/tests/interop/test_subscription_status_edge.py @@ -0,0 +1,25 @@ +import logging + +import pytest +from validatedpatterns_tests.interop import subscription + +from . import __loggername__ + +logger = logging.getLogger(__loggername__) + + +@pytest.mark.subscription_status_edge +def test_subscription_status_edge(openshift_dyn_client): + # These are the operator subscriptions and their associated namespaces + expected_subs = { + "openshift-gitops-operator": ["openshift-operators"], + } + + err_msg = subscription.subscription_status( + openshift_dyn_client, expected_subs, diff=False + ) + if err_msg: + logger.error(f"FAIL: {err_msg}") + assert False, err_msg + else: + logger.info("PASS: Subscription status check passed") diff --git a/tests/interop/test_subscription_status_hub.py b/tests/interop/test_subscription_status_hub.py new file mode 100644 index 0000000..90083a9 --- /dev/null +++ b/tests/interop/test_subscription_status_hub.py @@ -0,0 +1,27 @@ +import logging + +import pytest +from validatedpatterns_tests.interop import subscription + +from . import __loggername__ + +logger = logging.getLogger(__loggername__) + + +@pytest.mark.subscription_status_hub +def test_subscription_status_hub(openshift_dyn_client): + # These are the operator subscriptions and their associated namespaces + expected_subs = { + "openshift-gitops-operator": ["openshift-operators"], + "advanced-cluster-management": ["open-cluster-management"], + "multicluster-engine": ["multicluster-engine"], + } + + err_msg = subscription.subscription_status( + openshift_dyn_client, expected_subs, diff=True + ) + if err_msg: + logger.error(f"FAIL: {err_msg}") + assert False, err_msg + else: + logger.info("PASS: Subscription status check passed") diff --git a/tests/interop/test_validate_edge_site_components.py b/tests/interop/test_validate_edge_site_components.py new file mode 100644 index 0000000..3f4c91e --- /dev/null +++ b/tests/interop/test_validate_edge_site_components.py @@ -0,0 +1,72 @@ +import logging +import os + +import pytest +from validatedpatterns_tests.interop import application, components + +from . import __loggername__ + +logger = logging.getLogger(__loggername__) + +oc = os.environ["HOME"] + "/oc_client/oc" + + +@pytest.mark.test_validate_edge_site_components +def test_validate_edge_site_components(): + logger.info("Checking Openshift version on edge site") + version_out = components.dump_openshift_version() + logger.info(f"Openshift version:\n{version_out}") + + +@pytest.mark.validate_edge_site_reachable +def test_validate_edge_site_reachable(kube_config, openshift_dyn_client): + logger.info("Check if edge site API end point is reachable") + err_msg = components.validate_site_reachable(kube_config, openshift_dyn_client) + if err_msg: + logger.error(f"FAIL: {err_msg}") + assert False, err_msg + else: + logger.info("PASS: Edge site is reachable") + + +@pytest.mark.validate_argocd_reachable_edge_site +def test_validate_argocd_reachable_edge_site(openshift_dyn_client): + logger.info("Check if argocd route/url on edge site is reachable") + err_msg = components.validate_argocd_reachable(openshift_dyn_client) + if err_msg: + logger.error(f"FAIL: {err_msg}") + assert False, err_msg + else: + logger.info("PASS: Argocd is reachable") + + +@pytest.mark.check_pod_status_edge +def test_check_pod_status(openshift_dyn_client): + logger.info("Checking pod status") + projects = [ + "openshift-operators", + "open-cluster-management-agent", + "open-cluster-management-agent-addon", + "openshift-gitops", + ] + err_msg = components.check_pod_status(openshift_dyn_client, projects) + if err_msg: + logger.error(f"FAIL: {err_msg}") + assert False, err_msg + else: + logger.info("PASS: Pod status check succeeded.") + + +@pytest.mark.validate_argocd_applications_health_edge_site +def test_validate_argocd_applications_health_edge_site(openshift_dyn_client): + logger.info("Get all applications deployed by argocd on edge site") + projects = ["openshift-gitops"] + unhealthy_apps = application.get_argocd_application_status( + openshift_dyn_client, projects + ) + if unhealthy_apps: + err_msg = "Some or all applications deployed on edge site are unhealthy" + logger.error(f"FAIL: {err_msg}:\n{unhealthy_apps}") + assert False, err_msg + else: + logger.info("PASS: All applications deployed on edge site are healthy.") diff --git a/tests/interop/test_validate_hub_site_components.py b/tests/interop/test_validate_hub_site_components.py new file mode 100644 index 0000000..b5bec91 --- /dev/null +++ b/tests/interop/test_validate_hub_site_components.py @@ -0,0 +1,95 @@ +import logging +import os + +import pytest +from ocp_resources.storage_class import StorageClass +from validatedpatterns_tests.interop import application, components + +from . import __loggername__ + +logger = logging.getLogger(__loggername__) + +oc = os.environ["HOME"] + "/oc_client/oc" + + +@pytest.mark.test_validate_hub_site_components +def test_validate_hub_site_components(openshift_dyn_client): + logger.info("Checking Openshift version on hub site") + version_out = components.dump_openshift_version() + logger.info(f"Openshift version:\n{version_out}") + + logger.info("Dump PVC and storageclass info") + pvcs_out = components.dump_pvc() + logger.info(f"PVCs:\n{pvcs_out}") + + for sc in StorageClass.get(dyn_client=openshift_dyn_client): + logger.info(sc.instance) + + +@pytest.mark.validate_hub_site_reachable +def test_validate_hub_site_reachable(kube_config, openshift_dyn_client): + logger.info("Check if hub site API end point is reachable") + err_msg = components.validate_site_reachable(kube_config, openshift_dyn_client) + if err_msg: + logger.error(f"FAIL: {err_msg}") + assert False, err_msg + else: + logger.info("PASS: Hub site is reachable") + + +@pytest.mark.check_pod_status_hub +def test_check_pod_status(openshift_dyn_client): + logger.info("Checking pod status") + projects = [ + "openshift-operators", + "open-cluster-management", + "open-cluster-management-hub", + "openshift-gitops", + "vault", + ] + err_msg = components.check_pod_status(openshift_dyn_client, projects) + if err_msg: + logger.error(f"FAIL: {err_msg}") + assert False, err_msg + else: + logger.info("PASS: Pod status check succeeded.") + + +@pytest.mark.validate_acm_self_registration_managed_clusters +def test_validate_acm_self_registration_managed_clusters(openshift_dyn_client): + logger.info("Check ACM self registration for edge site") + kubefiles = [os.getenv("KUBECONFIG_EDGE")] + err_msg = components.validate_acm_self_registration_managed_clusters( + openshift_dyn_client, kubefiles + ) + if err_msg: + logger.error(f"FAIL: {err_msg}") + assert False, err_msg + else: + logger.info("PASS: Edge site is self registered") + + +@pytest.mark.validate_argocd_reachable_hub_site +def test_validate_argocd_reachable_hub_site(openshift_dyn_client): + logger.info("Check if argocd route/url on hub site is reachable") + err_msg = components.validate_argocd_reachable(openshift_dyn_client) + if err_msg: + logger.error(f"FAIL: {err_msg}") + assert False, err_msg + else: + logger.info("PASS: Argocd is reachable") + + +@pytest.mark.validate_argocd_applications_health_hub_site +def test_validate_argocd_applications_health_hub_site(openshift_dyn_client): + logger.info("Get all applications deployed by argocd on hub site") + projects = ["openshift-gitops", "multicloud-gitops-hub"] + unhealthy_apps = application.get_argocd_application_status( + openshift_dyn_client, projects + ) + if unhealthy_apps: + err_msg = "Some or all applications deployed on hub site are unhealthy" + logger.error(f"FAIL: {err_msg}:\n{unhealthy_apps}") + assert False, err_msg + else: + logger.info("PASS: All applications deployed on hub site are healthy.") diff --git a/values-4.18-hub.yaml b/values-4.18-hub.yaml new file mode 100644 index 0000000..faa4f2e --- /dev/null +++ b/values-4.18-hub.yaml @@ -0,0 +1,6 @@ +clusterGroup: + subscriptions: + acm: + name: advanced-cluster-management + namespace: open-cluster-management + channel: release-2.12 diff --git a/values-global.yaml b/values-global.yaml new file mode 100644 index 0000000..c01a319 --- /dev/null +++ b/values-global.yaml @@ -0,0 +1,12 @@ +--- +global: + pattern: multicloud-gitops + options: + useCSV: false + syncPolicy: Automatic + installPlanApproval: Automatic +main: + clusterGroupName: hub + multiSourceConfig: + enabled: true + clusterGroupChartVersion: "0.9.*" diff --git a/values-group-one.yaml b/values-group-one.yaml new file mode 100644 index 0000000..f17a726 --- /dev/null +++ b/values-group-one.yaml @@ -0,0 +1,108 @@ +global: + options: + useCSV: False + syncPolicy: Automatic + installPlanApproval: Automatic +clusterGroup: + name: group-one + isHubCluster: false + namespaces: + - config-demo + - hello-world + - golang-external-secrets + # The only subscription on spokes is gitops which gets managed by ACM + # subscriptions: + projects: + - eso + - config-demo + - hello-world + applications: + golang-external-secrets: + name: golang-external-secrets + namespace: golang-external-secrets + project: eso + chart: golang-external-secrets + chartVersion: 0.1.* + config-demo: + name: config-demo + namespace: config-demo + project: config-demo + path: charts/all/config-demo + hello-world: + name: hello-world + namespace: hello-world + project: hello-world + path: charts/all/hello-world + imperative: + # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm + # The default schedule is every 10 minutes: imperative.schedule + # Total timeout of all jobs is 1h: imperative.activeDeadlineSeconds + # imagePullPolicy is set to always: imperative.imagePullPolicy + # For additional overrides that apply to the jobs, please refer to + # https://hybrid-cloud-patterns.io/imperative-actions/#additional-job-customizations + jobs: + - name: hello-world + # ansible playbook to be run + playbook: rhvp.cluster_utils.hello_world + # per playbook timeout in seconds + timeout: 234 + # verbosity: "-v" + # Explicitly mention the cluster-state based overrides we plan to use for this pattern. + # We can use self-referential variables because the chart calls the tpl function with these variables defined + sharedValueFiles: + - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml' + # To mirror the "Classic" magic include structure, the clusterGroup would need all of these: + # sharedValueFiles: + # - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml' + # - '/overrides/values-{{ $.Values.global.clusterPlatform }}-{{ $.Values.global.clusterVersion }}.yaml' + # - '/overrides/values-{{ $.Values.global.clusterPlatform }}-{{ $.Values.clusterGroup.name }}.yaml' + # - '/overrides/values-{{ $.Values.global.clusterVersion }}-{{ $.Values.clusterGroup.name }}.yaml" +# To have apps in multiple flavors, use namespaces and use helm overrides as appropriate +# +# pipelines: +# name: pipelines +# namespace: production +# project: datacenter +# path: applications/pipeline +# repoURL: https://github.com/you/applications.git +# targetRevision: stable +# overrides: +# - name: myparam +# value: myparam +# +# pipelines_staging: +# - name: pipelines +# namespace: staging +# project: datacenter +# path: applications/pipeline +# repoURL: https://github.com/you/applications.git +# targetRevision: main +# +# Additional applications +# Be sure to include additional resources your apps will require +# +X machines +# +Y RAM +# +Z CPU +# vendor-app: +# name: vendor-app +# namespace: default +# project: vendor +# path: path/to/myapp +# repoURL: https://github.com/vendor/applications.git +# targetRevision: main + +# managedSites: +# factory: +# name: factory +# # repoURL: https://github.com/dagger-refuse-cool/manuela-factory.git +# targetRevision: main +# path: applications/factory +# helmOverrides: +# - name: site.isHubCluster +# value: false +# clusterSelector: +# matchExpressions: +# - key: vendor +# operator: In +# values: +# - OpenShift diff --git a/values-hub.yaml b/values-hub.yaml new file mode 100644 index 0000000..0850092 --- /dev/null +++ b/values-hub.yaml @@ -0,0 +1,171 @@ +clusterGroup: + name: hub + isHubCluster: true + namespaces: + - open-cluster-management + - vault + - golang-external-secrets + - config-demo + - hello-world + subscriptions: + acm: + name: advanced-cluster-management + namespace: open-cluster-management + channel: release-2.11 + #csv: advanced-cluster-management.v2.6.1 + projects: + - hub + - config-demo + - hello-world + # Explicitly mention the cluster-state based overrides we plan to use for this pattern. + # We can use self-referential variables because the chart calls the tpl function with these variables defined + sharedValueFiles: + - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml' + # sharedValueFiles is a flexible mechanism that will add the listed valuefiles to every app defined in the + # applications section. We intend this to supplement and possibly even replace previous "magic" mechanisms, though + # we do not at present have a target date for removal. + # + # To replicate the "classic" magic include structure, the clusterGroup would need all of these + # sharedValueFiles, in this order: + # - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml' + # - '/overrides/values-{{ $.Values.global.clusterPlatform }}-{{ $.Values.global.clusterVersion }}.yaml' + # - '/overrides/values-{{ $.Values.global.clusterPlatform }}-{{ $.Values.clusterGroup.name }}.yaml' + # - '/overrides/values-{{ $.Values.global.clusterVersion }}-{{ $.Values.clusterGroup.name }}.yaml" + # - '/overrides/values-{{ $.Values.global.localClusterName }}.yaml' + + # This kind of variable substitution will work with any of the variables the Validated Patterns operator knows + # about and sets, so this is also possible, for example: + # - '/overrides/values-{{ $.Values.global.hubClusterDomain }}.yaml' + # - '/overrides/values-{{ $.Values.global.localClusterDomain }}.yaml' + applications: + acm: + name: acm + namespace: open-cluster-management + project: hub + chart: acm + chartVersion: 0.1.* + ignoreDifferences: + - group: internal.open-cluster-management.io + kind: ManagedClusterInfo + jsonPointers: + - /spec/loggingCA + vault: + name: vault + namespace: vault + project: hub + chart: hashicorp-vault + chartVersion: 0.1.* + golang-external-secrets: + name: golang-external-secrets + namespace: golang-external-secrets + project: hub + chart: golang-external-secrets + chartVersion: 0.1.* + config-demo: + name: config-demo + namespace: config-demo + project: config-demo + path: charts/all/config-demo + hello-world: + name: hello-world + namespace: hello-world + project: hello-world + path: charts/all/hello-world + imperative: + # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm + # The default schedule is every 10 minutes: imperative.schedule + # Total timeout of all jobs is 1h: imperative.activeDeadlineSeconds + # imagePullPolicy is set to always: imperative.imagePullPolicy + # For additional overrides that apply to the jobs, please refer to + # https://hybrid-cloud-patterns.io/imperative-actions/#additional-job-customizations + jobs: + - name: hello-world + # ansible playbook to be run + playbook: rhvp.cluster_utils.hello_world + # per playbook timeout in seconds + timeout: 234 + # verbosity: "-v" + managedClusterGroups: + exampleRegion: + name: group-one + acmlabels: + - name: clusterGroup + value: group-one + helmOverrides: + - name: clusterGroup.isHubCluster + value: false + # Before enabling cluster provisioning, ensure AWS and/or Azure + # credentials and OCP pull secrets are defined in Vault. + # See values-secret.yaml.template + # + #clusterPools: + # exampleAWSPool: + # name: aws-ap + # openshiftVersion: 4.10.18 + # baseDomain: blueprints.rhecoeng.com + # platform: + # aws: + # region: ap-southeast-2 + # clusters: + # - One + # + # exampleAzurePool: + # name: azure-us + # openshiftVersion: 4.10.18 + # baseDomain: blueprints.rhecoeng.com + # platform: + # azure: + # baseDomainResourceGroupName: dojo-dns-zones + # region: eastus + # clusters: + # - Two + # - Three +# To have apps in multiple flavors, use namespaces and use helm overrides as appropriate +# +# pipelines: +# name: pipelines +# namespace: production +# project: datacenter +# path: applications/pipeline +# repoURL: https://github.com/you/applications.git +# targetRevision: stable +# overrides: +# - name: myparam +# value: myparam +# +# pipelines_staging: +# - name: pipelines +# namespace: staging +# project: datacenter +# path: applications/pipeline +# repoURL: https://github.com/you/applications.git +# targetRevision: main +# +# Additional applications +# Be sure to include additional resources your apps will require +# +X machines +# +Y RAM +# +Z CPU +# vendor-app: +# name: vendor-app +# namespace: default +# project: vendor +# path: path/to/myapp +# repoURL: https://github.com/vendor/applications.git +# targetRevision: main + +# managedSites: +# factory: +# name: factory +# # repoURL: https://github.com/dagger-refuse-cool/manuela-factory.git +# targetRevision: main +# path: applications/factory +# helmOverrides: +# - name: site.isHubCluster +# value: false +# clusterSelector: +# matchExpressions: +# - key: vendor +# operator: In +# values: +# - OpenShift diff --git a/values-secret.yaml.template b/values-secret.yaml.template new file mode 100644 index 0000000..e16735d --- /dev/null +++ b/values-secret.yaml.template @@ -0,0 +1,38 @@ +# A more formal description of this format can be found here: +# https://github.com/validatedpatterns/rhvp.cluster_utils/tree/main/roles/vault_utils#values-secret-file-format + +version: "2.0" +# Ideally you NEVER COMMIT THESE VALUES TO GIT (although if all passwords are +# automatically generated inside the vault this should not really matter) + +secrets: + - name: config-demo + vaultPrefixes: + - global + fields: + - name: secret + onMissingValue: generate + vaultPolicy: validatedPatternDefaultPolicy + + # If you use clusterPools you will need to uncomment the following lines + #- name: aws + # fields: + # - name: aws_access_key_id + # ini_file: ~/.aws/credentials + # ini_section: default + # ini_key: aws_access_key_id + # - name: aws_secret_access_key + # ini_file: ~/.aws/credentials + # ini_key: aws_secret_access_key + #- name: publickey + # fields: + # - name: content + # path: ~/.ssh/id_rsa.pub + #- name: privatekey + # fields: + # - name: content + # path: ~/.ssh/id_rsa + #- name: openshiftPullSecret + # fields: + # - name: content + # path: ~/.pullsecret.json diff --git a/values-standalone.yaml b/values-standalone.yaml new file mode 100644 index 0000000..41744ed --- /dev/null +++ b/values-standalone.yaml @@ -0,0 +1,61 @@ +clusterGroup: + name: standalone + isHubCluster: false + namespaces: + - vault + - golang-external-secrets + - config-demo + - hello-world + subscriptions: {} + projects: + - hub + - config-demo + - hello-world + # Explicitly mention the cluster-state based overrides we plan to use for this pattern. + # We can use self-referential variables because the chart calls the tpl function with these variables defined + sharedValueFiles: + - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml' + # sharedValueFiles is a flexible mechanism that will add the listed valuefiles to every app defined in the + # applications section. + # + # This kind of variable substitution will work with any of the variables the Validated Patterns operator knows + # about and sets, so this is also possible, for example: + # - '/overrides/values-{{ $.Values.global.hubClusterDomain }}.yaml' + # - '/overrides/values-{{ $.Values.global.localClusterDomain }}.yaml' + applications: + vault: + name: vault + namespace: vault + project: hub + chart: hashicorp-vault + chartVersion: 0.1.* + golang-external-secrets: + name: golang-external-secrets + namespace: golang-external-secrets + project: hub + chart: golang-external-secrets + chartVersion: 0.1.* + config-demo: + name: config-demo + namespace: config-demo + project: config-demo + path: charts/all/config-demo + hello-world: + name: hello-world + namespace: hello-world + project: hello-world + path: charts/all/hello-world + imperative: + # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm + # The default schedule is every 10 minutes: imperative.schedule + # Total timeout of all jobs is 1h: imperative.activeDeadlineSeconds + # imagePullPolicy is set to always: imperative.imagePullPolicy + # For additional overrides that apply to the jobs, please refer to + # https://hybrid-cloud-patterns.io/imperative-actions/#additional-job-customizations + jobs: + - name: hello-world + # ansible playbook to be run + playbook: rhvp.cluster_utils.hello_world + # per playbook timeout in seconds + timeout: 234 + # verbosity: "-v"