From 77fe43f37d19f80399299dcb02c0cb16efcf23ed Mon Sep 17 00:00:00 2001 From: kfox1111 Date: Fri, 18 Aug 2023 00:49:50 -0700 Subject: [PATCH] Cron job to check for and update images (#249) Co-authored-by: Marco Franssen --- .github/dependabot.yaml | 8 ++++ .github/scripts/edit-yaml.py | 24 ++++++++++ .github/scripts/requirements.txt | 2 + .github/scripts/update-tags.sh | 66 +++++++++++++++++++++++++++ .github/tests/images.json | 51 +++++++++++++++++++++ .github/workflows/check-versions.yaml | 20 ++++++++ 6 files changed, 171 insertions(+) create mode 100755 .github/scripts/edit-yaml.py create mode 100644 .github/scripts/requirements.txt create mode 100755 .github/scripts/update-tags.sh create mode 100644 .github/tests/images.json diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index c82e211..6a7cb06 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -12,3 +12,11 @@ updates: schedule: interval: "daily" open-pull-requests-limit: 5 +- package-ecosystem: pip + directory: "/.github/scripts" + schedule: + interval: daily + groups: + python-packages: + patterns: + - "*" diff --git a/.github/scripts/edit-yaml.py b/.github/scripts/edit-yaml.py new file mode 100755 index 0000000..2709e77 --- /dev/null +++ b/.github/scripts/edit-yaml.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +import os +import sys +from dict_deep import deep_set +import ruamel.yaml + +def represent_none(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:null', u'null') + +y = ruamel.yaml.YAML() +y.indent(mapping=2, sequence=4, offset=2) +# Dont wrap long lines +y.width = 4096 +y.preserve_quotes = True +y.representer.add_representer(type(None), represent_none) + +d = y.load(open(os.environ['VALUES'])) + +tagquery = os.environ['QUERY'] + '.tag' + +deep_set(d, tagquery, os.environ['LATEST_VERSION']); + +y.dump(d, sys.stdout) diff --git a/.github/scripts/requirements.txt b/.github/scripts/requirements.txt new file mode 100644 index 0000000..f139ec1 --- /dev/null +++ b/.github/scripts/requirements.txt @@ -0,0 +1,2 @@ +ruamel.yaml +dict_deep diff --git a/.github/scripts/update-tags.sh b/.github/scripts/update-tags.sh new file mode 100755 index 0000000..4ba69e5 --- /dev/null +++ b/.github/scripts/update-tags.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +SCRIPT="$(readlink -f "$0")" +SCRIPTPATH="$(dirname "${SCRIPT}")" + +IMAGEJSON="${SCRIPTPATH}/../tests/images.json" + +if ! command -v crane &> /dev/null; then + echo Please install crane + exit 1 +fi + +if ! command -v jq &> /dev/null; then + echo Please install jq + exit 1 +fi + +if ! command -v yq &> /dev/null; then + echo Please install yq + exit 1 +fi + +if ! command -v python3 -c 'import ruamel.yaml' &> /dev/null; then + echo Please install python3 with the ruamel.yaml module + exit 1 +fi + +if ! command -v python3 -c 'import dict_deep' &> /dev/null; then + echo Please install python3 with the dict_deep module + exit 1 +fi + +jq -r '. | keys[]' "$IMAGEJSON" | while read -r CHART; do + jq -r ".\"${CHART}\" | keys[]" "$IMAGEJSON" | while read -r IDX; do + QUERY=$(jq -r ".\"${CHART}\"[${IDX}].query" "$IMAGEJSON") + FILTER=$(jq -r ".\"${CHART}\"[${IDX}].filter" "$IMAGEJSON") + + OLD_IFS=${IFS} + SORTFLAGS=() + while IFS='' read -r value; do + SORTFLAGS+=("$value") + done < <(jq -r ".\"${CHART}\"[${IDX}].\"sort-flags\" | .[]" "$IMAGEJSON") + IFS=${OLD_IFS} + + VALUES="${SCRIPTPATH}/../../charts/spire/charts/${CHART}" + REGISTRY=$(yq e ".${QUERY}.registry" "$VALUES") + REPOSITORY=$(yq e ".${QUERY}.repository" "$VALUES") + VERSION=$(yq e ".${QUERY}.tag" "$VALUES") + if [[ "$FILTER" == "LATESTSHA" ]]; then + LATEST_VERSION="latest@"$(crane digest "${REGISTRY}/${REPOSITORY}:latest") + else + LATEST_VERSION=$(crane ls "${REGISTRY}/${REPOSITORY}" | grep "${FILTER}" | sort "${SORTFLAGS[@]}"| tail -n 1) + fi + + export QUERY + export VALUES + export LATEST_VERSION + + if [ "${VERSION}" != "${LATEST_VERSION}" ]; then + echo "New image version found: ${REGISTRY}/${REPOSITORY}:${LATEST_VERSION}" + "${SCRIPTPATH}/edit-yaml.py" > /tmp/$$ + mv /tmp/$$ "${VALUES}" + fi + done +done +"${SCRIPTPATH}/../../helm-docs.sh" diff --git a/.github/tests/images.json b/.github/tests/images.json new file mode 100644 index 0000000..d33d64c --- /dev/null +++ b/.github/tests/images.json @@ -0,0 +1,51 @@ +{ + "spire-server/values.yaml": [ + { + "query": "tests.bash.image", + "filter": "LATESTSHA", + "sort-flags": [] + } + ], + "spire-agent/values.yaml": [ + { + "query": "waitForIt.image", + "filter": "LATESTSHA", + "sort-flags": [] + }, + { + "query": "fsGroupFix.image", + "filter": "LATESTSHA", + "sort-flags": [] + } + ], + "spiffe-csi-driver/values.yaml": [ + { + "query": "nodeDriverRegistrar.image", + "filter": "^v", + "sort-flags": [] + } + ], + "spiffe-oidc-discovery-provider/values.yaml": [ + { + "query": "insecureScheme.nginx.image", + "filter": "^[0-9]\\+\\.[0-9]\\+\\.[0-9]\\+-alpine$", + "sort-flags": [] + }, { + "query": "telemetry.prometheus.nginxExporter.image", + "filter": "^[0-9]\\+\\.[0-9]\\+\\.[0-9]\\+$", + "sort-flags": ["-t", ".", "-k1,1n", "-k2,2n", "-k3,3n"] + }, + { + "query": "tests.bash.image", + "filter": "LATESTSHA", + "sort-flags": [] + } + ], + "tornjak-frontend/values.yaml": [ + { + "query": "tests.bash.image", + "filter": "LATESTSHA", + "sort-flags": [] + } + ] +} diff --git a/.github/workflows/check-versions.yaml b/.github/workflows/check-versions.yaml index 913808c..305c1c3 100644 --- a/.github/workflows/check-versions.yaml +++ b/.github/workflows/check-versions.yaml @@ -32,6 +32,26 @@ jobs: ./.github/scripts/update-versions.sh git diff + - name: Setup go + uses: actions/setup-go@v4.0.0 + + - name: Setup crane + uses: imjasonh/setup-crane@v0.1 + + - uses: actions/setup-python@v4.6.1 + with: + python-version: '3.9' + + - name: Update image tags + run: | + go install github.com/mikefarah/yq/v4@latest + + sudo apt-get install wget apt-transport-https gnupg lsb-release + sudo pip install -r .github/scripts/requirements.txt + + ./.github/scripts/update-tags.sh + git diff + - name: Create Pull Request id: cpr uses: peter-evans/create-pull-request@v5.0.2