diff --git a/config/templates/hive-csv-template.yaml b/config/templates/hive-csv-template.yaml index 8be52734acd..de6d14ebe64 100644 --- a/config/templates/hive-csv-template.yaml +++ b/config/templates/hive-csv-template.yaml @@ -108,10 +108,10 @@ spec: spec: clusterPermissions: - serviceAccountName: hive-operator - # Rules will be added here by the bundle-gen.py script. + # Rules will be added here by the bundle-gen.sh script. deployments: - name: hive-operator - # Deployment spec will be added here by the bundle-gen.py script. + # Deployment spec will be added here by the bundle-gen.sh script. customresourcedefinitions: owned: - description: Configuration for the Hive Operator diff --git a/hack/app-sre/generate-saas-template.sh b/hack/app-sre/generate-saas-template.sh index 0d57bb94866..241edbfb442 100755 --- a/hack/app-sre/generate-saas-template.sh +++ b/hack/app-sre/generate-saas-template.sh @@ -22,19 +22,20 @@ Parameters: EOF } -# Check for help flag or incorrect number of arguments -if [ $# -ne 3 ]; then - usage - exit 1 -fi - +# Check for help flag for arg in "$@"; do - if [[ "$arg" == *"-h"* ]]; then + if [[ "$arg" == "-h" || "$arg" == "--help" ]]; then usage - exit 1 + exit 0 fi done +# Check for incorrect number of arguments +if [ $# -ne 3 ]; then + usage + exit 1 +fi + saas_template_stub="$1" saas_object_file="$2" out_file="$3" diff --git a/hack/bundle-gen.py b/hack/bundle-gen.py deleted file mode 100755 index 2c26a33ff5e..00000000000 --- a/hack/bundle-gen.py +++ /dev/null @@ -1,653 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import datetime -import git -import github as gh -import json -import os -import requests -import semver -import shutil -import subprocess -import sys -import tempfile -import urllib3 -import yaml - -import version2 - -HIVE_REPO_DEFAULT = "git@github.com:openshift/hive.git" - -# Hive dir within both: -# https://github.com/redhat-openshift-ecosystem/community-operators-prod -# https://github.com/k8s-operatorhub/community-operators -HIVE_SUB_DIR = "operators/hive-operator" - -OPERATORHUB_HIVE_IMAGE_DEFAULT = "quay.io/openshift-hive/hive" - -COMMUNITY_OPERATORS_UPSTREAM_REPO = "git@github.com:redhat-openshift-ecosystem/community-operators-prod.git" - -SUBPROCESS_REDIRECT = subprocess.DEVNULL - -HIVE_BRANCH_DEFAULT = "master" - -CHANNEL_DEFAULT = "alpha" - - -def get_params(): - parser = argparse.ArgumentParser( - description="Hive Bundle Generator and Publishing Script", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=f""" -This utility will: -1. Clone the repo specified by --hive-repo (default: {HIVE_REPO_DEFAULT}). -2. Check out the commit-ish specified by --commit (default: the tip of - {HIVE_BRANCH_DEFAULT}). -3. Look for a hive image in the repo specified by --image-repo (default - {OPERATORHUB_HIVE_IMAGE_DEFAULT}) with the tag specified by --image-tag-override - (default: 10-character SHA of the checked-out commit). - - If not found, build and push that image. - - You can skip this step by providing --skip-image-validation. -4. Generate an OperatorHub bundle from the checked-out commit, pointing to the image - validated and/or built and pushed above. -5. Open PRs with this bundle in the Red Hat and upstream Kubernetes community operator - projects: - - github.com/redhat-openshift-ecosystem/community-operators-prod - - github.com/k8s-operatorhub/community-operators -""" - ) - parser.add_argument( - "--verbose", - default=False, - help="Show more details while running", - action="store_true", - ) - parser.add_argument( - "--dry-run", - default=False, - help="""Test run that skips building/pushing images, pushing branches, -and submitting PRs""", - action="store_true", - ) - # TODO: Validate this early! As written, if this is wrong you won't bounce until open_pr. - parser.add_argument( - "--github-user", - default=os.getenv("GITHUB_USER") or os.environ["USER"], - help="User's github username. Defaults to $GITHUB_USER, then $USER.", - ) - parser.add_argument( - "--hold", - default=False, - help="""Adds a /hold comment in commit body to prevent the PR from -merging (use "/hold cancel" to remove)""", - action="store_true", - ) - parser.add_argument( - "--hive-repo", - default=HIVE_REPO_DEFAULT, - help="""The hive git repository to clone. E.g. save time by using a local -directory (but make sure it's up to date!)""", - ) - parser.add_argument( - "--skip-release-config", - default=False, - help="""release-config.yaml file generated by default is needed for -bundles to be deployed on the OperatorHubs. This argument can be used -to skip the creation of this file""", - action="store_true" - ) - parser.add_argument( - "--image-repo", - default=OPERATORHUB_HIVE_IMAGE_DEFAULT, - help="""The image repository housing the operator image, e.g. -`quay.io/myproject/hive`. (Do not include a tag; it will be -generated based on the branch/commit.)""", - ) - parser.add_argument( - "--image-tag-override", - help="""String to use for the image tag. By default we use the first ten -digits of the sha corresponding to the requested COMMIT-ISH.""", - ) - parser.add_argument( - "--commit", - metavar="COMMIT-ISH", - help="""The commit-ish from which to build and push the image. -This argument may be used on its own or in conjunction with ---dummy-bundle. If used as a standalone option, we generate a -bundle version for `{}` starting on the commit-ish provided. -If --commit is not specified, we assume the tip of the {} -branch. If used alongside --dummy-bundle, we generate bundle -files for the `mce-X.Y` branch specified, at the commit-ish -specified. For example, `--dummy-bundle mce-2.1 --commit $sha -will result in a dummy-bundle 2.1.$count-$sha for mce-2.1. -For more information, see the --dummy-bundle description.""".format( - HIVE_BRANCH_DEFAULT, - HIVE_BRANCH_DEFAULT, - ), - ) - parser.add_argument( - "--dummy-bundle", - metavar="BRANCH-NAME", - help="""Only generate bundle files at a specific commit, as provided by -the `--commit` parameter, defaulting to the commit corresponding -to BRANCH-NAME, which must be `{}` or a valid `mce-*` branch. -The bundle files will be placed in a subdirectory of your PWD -named hive-operator-bundle-$version, where $version is computed -as 'X.Y.$count-$sha'; X.Y is the MCE version number, or `{}` for -{}; $count is the number of commits leading up to the -requested commit; and $sha is the first seven digits of the SHA -of the requested commit. No package file will be generated. The -CSV will not have any graph directives (`replaces`, `skipRange`, -etc.).""".format( - HIVE_BRANCH_DEFAULT, - version2.MASTER_BRANCH_PREFIX, - HIVE_BRANCH_DEFAULT, - ), - ) - parser.add_argument( - "--skip-image-validation", - default=False, - help="""By default, we will check to make sure the image described by ---image-repo and --image-tag-override (both of which may be -defaulted/computed) exists, building and pushing it if it does -not. Provide this flag to skip that validation. Also note that we -will currently only validate quay.io/* images.""", - action="store_true", - ) - args = parser.parse_args() - - if args.verbose: - global SUBPROCESS_REDIRECT - SUBPROCESS_REDIRECT = None - - return args - -# Traverse through version directories to detect highest version -def get_previous_version(work_dir, channel_name): - upstream_branch = "main" - dir_name = "community-operators-prod" - - repo_full_path = os.path.join(work_dir, dir_name) - # clone git repo - try: - git.Repo.clone_from(COMMUNITY_OPERATORS_UPSTREAM_REPO, repo_full_path) - except: - print("Failed to clone repo {} to {}".format(COMMUNITY_OPERATORS_UPSTREAM_REPO, repo_full_path)) - raise - - # get to the right place on the filesystem - print("Working in %s" % repo_full_path) - os.chdir(repo_full_path) - - repo = git.Repo(repo_full_path) - - # Starting branch - print("Checkout latest {}".format(upstream_branch)) - try: - repo.git.checkout(upstream_branch) - except: - print("Failed to checkout {}".format(upstream_branch)) - raise - - highest_version = "0.0.0" - try: - hive_dir = os.path.join(repo_full_path, HIVE_SUB_DIR) - for version in os.listdir(hive_dir): - annotation_yaml_path = os.path.join(hive_dir, version, "metadata", "annotations.yaml") - try: - with open(annotation_yaml_path, "r") as stream: - annotation_yaml = yaml.load(stream, Loader=yaml.SafeLoader) - version_channels = annotation_yaml["annotations"]["operators.operatorframework.io.bundle.channels.v1"] - if channel_name in version_channels.split(","): - if semver.compare(version, highest_version) > 0: - highest_version = version - except (NotADirectoryError, FileNotFoundError): - print("Skipping %s -- not a version", version) - continue - except: - print( - "Unable to determine previous hive version from {}", - COMMUNITY_OPERATORS_UPSTREAM_REPO, - ) - raise - - if highest_version == "0.0.0": - # Channel not found -- no previous version. - # NOTE: If a new channel is introduced, finding a prev_version will fail and require a manual edit of the CSV with the prev_version. - # Keeping this condition fatal in order to ensure that a failure to find prev_version is noticed. - print( - "Channel not found. Unable to calculate determine version {}", - COMMUNITY_OPERATORS_UPSTREAM_REPO, - ) - sys.exit(1) - - return highest_version - - -# generate_csv_base generates a hive bundle from the current working directory -# and deposits all artifacts in the specified bundle_dir. -# If prev_version is not None/empty, the CSV will include it as `replaces`. -def generate_csv_base( - bundle_dir, image_repo, v: version2.Version, prev_version, image_tag, skip_release_config -): - print("Writing bundle files to directory: %s" % bundle_dir) - print("Generating CSV for version: %s" % v) - - crds_dir = "config/crds" - csv_template = "config/templates/hive-csv-template.yaml" - operator_role = "config/operator/operator_role.yaml" - deployment_spec = "config/operator/operator_deployment.yaml" - - - # The bundle directory doesn't have the 'v' - version_dir = os.path.join(bundle_dir, v.semver) - if not os.path.exists(version_dir): - os.mkdir(version_dir) - manifests_dir = os.path.join(version_dir, "manifests") - if not os.path.exists(manifests_dir): - os.mkdir(manifests_dir) - - # Create annotations.yaml file - metadata_dir = os.path.join(version_dir, "metadata") - if not os.path.exists(metadata_dir): - os.mkdir(metadata_dir) - file_path = os.path.join(metadata_dir, "annotations.yaml") - with open(file_path, 'w') as file: - yaml_annotations = { - 'annotations' : { - 'operators.operatorframework.io.bundle.channel.default.v1': CHANNEL_DEFAULT, - 'operators.operatorframework.io.bundle.channels.v1': CHANNEL_DEFAULT, - 'operators.operatorframework.io.bundle.manifests.v1': 'manifests/', - 'operators.operatorframework.io.bundle.mediatype.v1': 'registry+v1', - 'operators.operatorframework.io.bundle.metadata.v1': 'metadata/', - 'operators.operatorframework.io.bundle.package.v1': 'hive-operator', - } - } - - yaml_string = yaml.dump(yaml_annotations) - file.write(yaml_string) - - if not skip_release_config: - # Create release-config.yaml file. This file is only utilized by the Red Hat Ecosystem, - # for the automatic catalog update for all RH catalogs. Kubernetes Ecosystem will ignore the file. - file_path = os.path.join(version_dir, "release-config.yaml") - data = { - "catalog_templates": [ - { - "channels": [CHANNEL_DEFAULT], - "replaces": "hive-operator.v"+prev_version, - "template_name": "basic.yaml" - } - ] - } - with open(file_path, 'w') as file: - yaml_string = yaml.dump(data) - file.write(yaml_string) - - owned_crds = [] - - # Copy all CSV files over to the manifests dir: - crd_files = sorted(os.listdir(crds_dir)) - for file_name in crd_files: - full_path = os.path.join(crds_dir, file_name) - if os.path.isfile(os.path.join(crds_dir, file_name)): - dest_path = os.path.join(manifests_dir, file_name) - shutil.copy(full_path, dest_path) - # Read the CRD yaml to add to owned CRDs list - with open(dest_path, "r") as stream: - crd_csv = yaml.load(stream, Loader=yaml.SafeLoader) - owned_crds.append( - { - "description": crd_csv["spec"]["versions"][0]["schema"][ - "openAPIV3Schema" - ]["description"], - "displayName": crd_csv["spec"]["names"]["kind"], - "kind": crd_csv["spec"]["names"]["kind"], - "name": crd_csv["metadata"]["name"], - "version": crd_csv["spec"]["versions"][0]["name"], - } - ) - - with open(csv_template, "r") as stream: - csv = yaml.load(stream, Loader=yaml.SafeLoader) - - csv["spec"]["customresourcedefinitions"]["owned"] = owned_crds - - csv["spec"]["install"]["spec"]["clusterPermissions"] = [] - - # Add our operator role to the CSV: - with open(operator_role, "r") as stream: - operator_role = yaml.load(stream, Loader=yaml.SafeLoader) - csv["spec"]["install"]["spec"]["clusterPermissions"].append( - { - "rules": operator_role["rules"], - "serviceAccountName": "hive-operator", - } - ) - - # Add our deployment spec for the hive operator: - with open(deployment_spec, "r") as stream: - operator_components = [] - operator = yaml.load_all(stream, Loader=yaml.SafeLoader) - for doc in operator: - operator_components.append(doc) - operator_deployment = operator_components[1] - csv["spec"]["install"]["spec"]["deployments"][0]["spec"] = operator_deployment[ - "spec" - ] - - # Update the versions to include git hash: - csv["metadata"]["name"] = "hive-operator.%s" % v - csv["spec"]["version"] = v.semver - if prev_version: - csv["spec"]["replaces"] = "hive-operator.v%s" % prev_version - - # Update the deployment to use the defined image: - image_ref = "%s:%s" % (image_repo, image_tag) - csv["spec"]["install"]["spec"]["deployments"][0]["spec"]["template"]["spec"][ - "containers" - ][0]["image"] = image_ref - csv["metadata"]["annotations"]["containerImage"] = image_ref - - # Set the CSV createdAt annotation: - now = datetime.datetime.now() - csv["metadata"]["annotations"]["createdAt"] = now.strftime("%Y-%m-%dT%H:%M:%SZ") - - # Write the CSV to disk: - csv_filename = "hive-operator.%s.clusterserviceversion.yaml" % v - csv_file = os.path.join(manifests_dir, csv_filename) - with open(csv_file, "w") as outfile: - yaml.dump(csv, outfile, default_flow_style=False) - print("Wrote ClusterServiceVersion: %s" % csv_file) - - return version_dir - - -def validate_image(image_repo, image_tag, skip): - """Ensure the image exists. - - We only attempt to validate quay.io images. - - :param image_repo: The `host/org/repo` of the image. - :param image_tag: The tag of the image. - :param skip: If True, we'll skip validation. - :return bool: True if a) the image exists; or b) we skipped validation. - False if we determine the image does not exist. - """ - if skip: - print(f"Skipping validation of image {image_repo}:{image_tag}") - return True - parsed = urllib3.util.parse_url(image_repo) - if parsed.host != "quay.io": - print(f"Skipping validation of non-quay image in repo {image_repo}") - return True - url = f"https://{parsed.host}/api/v1/repository{parsed.path}/tag/?specificTag={image_tag}" - resp = requests.get(url) - if not resp.ok: - print( - f"Failed to validate image {image_repo}:{image_tag}!\nCouldn't query quay API for {url}\n\tstatus_code={resp.status_code}" - ) - sys.exit(1) - j = resp.json() - if not j.get("tags"): - print(f"No image at {image_repo}:{image_tag}!") - return False - - print("Image validated successfully") - return True - - -def generate_package(package_file, channel, v: version2.Version): - document_template = """ - channels: - - currentCSV: %s - name: %s - defaultChannel: %s - packageName: hive-operator -""" - name = "hive-operator.%s" % v - document = document_template % (name, channel, channel) - - with open(package_file, "w") as outfile: - yaml.dump( - yaml.load(document, Loader=yaml.SafeLoader), - outfile, - default_flow_style=False, - ) - print("Wrote package: %s" % package_file) - - -def copy_bundle(orig_wd, bundle_source_dir, v: version2.Version): - bundle_dest_dir = os.path.join(orig_wd, "hive-operator-bundle-{}".format(str(v))) - shutil.copytree(bundle_source_dir, bundle_dest_dir) - print("Wrote bundle to {}".format(bundle_dest_dir)) - - -def open_pr( - work_dir, - fork_repo, - upstream_repo, - gh_username, - bundle_source_dir, - v: version2.Version, - hold, - dry_run, -): - dir_name = fork_repo.split("/")[1][:-4] - - dest_github_org = upstream_repo.split(":")[1].split("/")[0] - dest_github_reponame = dir_name - - os.chdir(work_dir) - - print() - print() - repo_full_path = os.path.join(work_dir, dir_name) - # The get_previous_version function clones the upstream RH directory - # into community-operators-prod repo. - # If this repo already exists, skip cloning it a second time. - if not os.path.exists(repo_full_path): - # clone git repo - print("Cloning %s" % fork_repo) - try: - git.Repo.clone_from(fork_repo, repo_full_path) - except: - print("Failed to clone repo {} to {}".format(fork_repo, repo_full_path)) - raise - else: - print("Skipping cloning of %s. Repo already exists" % fork_repo) - - # get to the right place on the filesystem - print("Working in %s" % repo_full_path) - os.chdir(repo_full_path) - - repo = git.Repo(repo_full_path) - - try: - repo.remotes.origin.set_url(fork_repo) - except: - print("Failed to set origin remote") - raise - - try: - repo.create_remote("upstream", upstream_repo) - except: - print("Failed to create upstream remote") - raise - - print("Fetching latest upstream") - try: - repo.remotes.upstream.fetch() - except: - print("Failed to fetch upstream") - raise - - # Starting branch - print("Checkout latest upstream/main") - try: - repo.git.checkout("upstream/main") - except: - print("Failed to checkout upstream/main") - raise - - branch_name = "update-hive-{}".format(v.semver) - - print("Create branch {}".format(branch_name)) - try: - repo.git.checkout("-b", branch_name) - except: - print("Failed to checkout branch {}".format(branch_name)) - raise - - # copy bundle directory - print("Copying bundle directory") - bundle_files = os.path.join(bundle_source_dir, v.semver) - hive_dir = os.path.join(repo_full_path, HIVE_SUB_DIR, v.semver) - shutil.copytree(bundle_files, hive_dir) - - pr_title = "operator hive-operator ({})".format(v.semver) - - # commit files - print("Adding file") - repo.git.add(HIVE_SUB_DIR) - - print("Committing {}".format(pr_title)) - try: - repo.git.commit("--signoff", "--message={}".format(pr_title)) - except: - print("Failed to commit") - raise - print() - - if not dry_run: - print("Pushing branch {}".format(branch_name)) - origin = repo.remotes.origin - try: - origin.push(branch_name, None, force=True) - except: - print("failed to push branch to origin") - raise - - # open PR - client = gh.GitHubClient(dest_github_org, dest_github_reponame, "") - - from_branch = "{}:{}".format(gh_username, branch_name) - to_branch = "main" - - body = pr_title - if hold: - body = "%s\n\n/hold" % body - - resp = client.create_pr(from_branch, to_branch, pr_title, body) - if resp.status_code != 201: # 201 == Created - print(resp.text) - sys.exit(1) - - json_content = json.loads(resp.content.decode("utf-8")) - print("PR opened: {}".format(json_content["html_url"])) - - else: - print("Skipping branch push due to dry-run") - print() - - -def build_image(uri, dry): - if dry: - print(f"DRY RUN: Skipping building image {uri}") - return True - - print(f"Building image {uri}") - return subprocess.run(['make', f'IMG={uri}', 'podman-operatorhub-build']).returncode == 0 - - -def push_image(uri, dry): - if dry: - print(f"DRY RUN: Skipping pushing image {uri}") - return True - - print(f"Pushing image {uri}") - return subprocess.run(['podman', 'push', uri]).returncode == 0 - - -if __name__ == "__main__": - args = get_params() - - hive_repo_dir = tempfile.TemporaryDirectory(prefix="hive-repo-") - - print("Cloning {} to {}".format(args.hive_repo, hive_repo_dir.name)) - try: - git.Repo.clone_from(args.hive_repo, hive_repo_dir.name) - except: - print( - "Failed to clone repo {} to {}".format(args.hive_repo, hive_repo_dir.name) - ) - raise - - bundle_dir = tempfile.TemporaryDirectory(prefix="hive-operator-bundle-") - work_dir = tempfile.TemporaryDirectory(prefix="operatorhub-push-") - - orig_wd = os.getcwd() - print("Working in {}".format(hive_repo_dir.name)) - os.chdir(hive_repo_dir.name) - - ver = version2.Version( - hive_repo_dir.name, branch_name=args.dummy_bundle, commit_ish=args.commit - ) - print("Checking out {}".format(ver.shortsha)) - try: - ver.repo.git.checkout(ver.commit) - except: - print("Failed to checkout {}".format(ver.shortsha)) - raise - - image_tag = args.image_tag_override or ver.commit.hexsha[0:10] - if not validate_image(args.image_repo, image_tag, args.skip_image_validation): - uri = f'{args.image_repo}:{image_tag}' - if not build_image(uri, args.dry_run): - sys.exit(1) - if not push_image(uri, args.dry_run): - sys.exit(1) - - if args.dummy_bundle: - # Omit version graph stuff - prev_version = None - else: - prev_version = get_previous_version(work_dir.name, CHANNEL_DEFAULT) - os.chdir(hive_repo_dir.name) - if ver.semver == prev_version: - raise ValueError("Version {} already exists upstream".format(ver.semver)) - - version_dir = generate_csv_base( - bundle_dir.name, args.image_repo, ver, prev_version, image_tag, args.skip_release_config - ) - - if args.dummy_bundle: - copy_bundle(orig_wd, version_dir, ver) - else: - # redhat-openshift-ecosystem/community-operators-prod - open_pr( - work_dir.name, - "git@github.com:%s/community-operators-prod.git" % args.github_user, - "git@github.com:redhat-openshift-ecosystem/community-operators-prod.git", - args.github_user, - bundle_dir.name, - ver, - args.hold, - args.dry_run, - ) - # k8s-operatorhub/community-operators - open_pr( - work_dir.name, - "git@github.com:%s/community-operators.git" % args.github_user, - "git@github.com:k8s-operatorhub/community-operators.git", - args.github_user, - bundle_dir.name, - ver, - args.hold, - args.dry_run, - ) - - hive_repo_dir.cleanup() - bundle_dir.cleanup() - work_dir.cleanup() diff --git a/hack/bundle-gen.sh b/hack/bundle-gen.sh new file mode 100755 index 00000000000..1cd3daa23a7 --- /dev/null +++ b/hack/bundle-gen.sh @@ -0,0 +1,550 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +# shellcheck source=hack/version2.sh +source "$SCRIPT_DIR/version2.sh" + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- +readonly HIVE_REPO_DEFAULT="git@github.com:openshift/hive.git" +# Hive dir within both: +# https://github.com/redhat-openshift-ecosystem/community-operators-prod +# https://github.com/k8s-operatorhub/community-operators +readonly HIVE_SUB_DIR="operators/hive-operator" +readonly IMAGE_REPO_DEFAULT="quay.io/openshift-hive/hive" +readonly COMMUNITY_OPERATORS_UPSTREAM="${COMMUNITY_OPERATORS_UPSTREAM:-git@github.com:redhat-openshift-ecosystem/community-operators-prod.git}" +readonly CHANNEL_DEFAULT="alpha" + +# --------------------------------------------------------------------------- +# Runtime state — set by parse_args +# --------------------------------------------------------------------------- +VERBOSE=false +DRY_RUN=false +HOLD=false +SKIP_RELEASE_CONFIG=false +SKIP_IMAGE_VALIDATION=false +GITHUB_USER="${GITHUB_USER:-${USER}}" +HIVE_REPO="$HIVE_REPO_DEFAULT" +IMAGE_REPO="$IMAGE_REPO_DEFAULT" +IMAGE_TAG_OVERRIDE="" +COMMIT_ISH="" +DUMMY_BUNDLE="" + +# Temp dirs — populated in main, removed by cleanup trap +HIVE_REPO_DIR="" +BUNDLE_DIR="" +WORK_DIR="" + +# --------------------------------------------------------------------------- +# usage +# --------------------------------------------------------------------------- +usage() { + cat <&2 + usage >&2 + exit 1 + fi +} + +# --------------------------------------------------------------------------- +# parse_args +# --------------------------------------------------------------------------- +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --verbose) VERBOSE=true ;; + --dry-run) DRY_RUN=true ;; + --hold) HOLD=true ;; + --skip-release-config) SKIP_RELEASE_CONFIG=true ;; + --skip-image-validation) SKIP_IMAGE_VALIDATION=true ;; + # TODO: Validate this early — if wrong you won't bounce until open_pr. + --github-user) require_arg "$1" "${2:-}"; GITHUB_USER="$2"; shift ;; + --hive-repo) require_arg "$1" "${2:-}"; HIVE_REPO="$2"; shift ;; + --image-repo) require_arg "$1" "${2:-}"; IMAGE_REPO="$2"; shift ;; + --image-tag-override) require_arg "$1" "${2:-}"; IMAGE_TAG_OVERRIDE="$2"; shift ;; + --commit) require_arg "$1" "${2:-}"; COMMIT_ISH="$2"; shift ;; + --dummy-bundle) require_arg "$1" "${2:-}"; DUMMY_BUNDLE="$2"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; + esac + shift + done +} + +# --------------------------------------------------------------------------- +# cleanup — registered as EXIT trap +# --------------------------------------------------------------------------- +cleanup() { + [[ -n "$HIVE_REPO_DIR" ]] && rm -rf "$HIVE_REPO_DIR" + [[ -n "$BUNDLE_DIR" ]] && rm -rf "$BUNDLE_DIR" + [[ -n "$WORK_DIR" ]] && rm -rf "$WORK_DIR" +} + +# --------------------------------------------------------------------------- +# validate_image +# Returns 0 if the image exists or validation is skipped; 1 if not found. +# --------------------------------------------------------------------------- +validate_image() { + local image_repo="$1" image_tag="$2" + + if $SKIP_IMAGE_VALIDATION; then + echo "Skipping image validation for ${image_repo}:${image_tag}" + return 0 + fi + + # Only validate quay.io images + local host="${image_repo%%/*}" + if [[ "$host" != "quay.io" ]]; then + echo "Skipping validation for non-quay image: ${image_repo}:${image_tag}" + return 0 + fi + + local path="${image_repo#*/}" + local url="https://quay.io/api/v1/repository/${path}/tag/?specificTag=${image_tag}" + + local resp + if ! resp=$(curl -sf --max-time 30 "$url"); then + echo "Failed to query quay API: $url" >&2 + exit 1 + fi + + if [[ "$(echo "$resp" | jq '.tags | length')" -eq 0 ]]; then + echo "No image found at ${image_repo}:${image_tag}" + return 1 + fi + + echo "Image validated: ${image_repo}:${image_tag}" +} + +# --------------------------------------------------------------------------- +# ensure_image — validates; builds and pushes if missing. +# Must be called from the hive repo root (make target lives there). +# --------------------------------------------------------------------------- +ensure_image() { + local image_repo="$1" image_tag="$2" + local uri="${image_repo}:${image_tag}" + + validate_image "$image_repo" "$image_tag" && return 0 + + if $DRY_RUN; then + echo "DRY RUN: skipping build/push of $uri" + return 0 + fi + + echo "Building image $uri" + make "IMG=$uri" podman-operatorhub-build || { echo "Image build failed" >&2; exit 1; } + + echo "Pushing image $uri" + podman push "$uri" || { echo "Image push failed" >&2; exit 1; } +} + +# --------------------------------------------------------------------------- +# semver_gt — returns 0 if $1 > $2 using version-aware sort. +# Strips the git-hash suffix (everything after the first '-') before comparing +# so that e.g. "1.2.3200-abc1234" > "1.2.3187-18827f6". +# --------------------------------------------------------------------------- +semver_gt() { + local a="${1%%-*}" b="${2%%-*}" + [[ "$a" != "$b" ]] && \ + [[ "$(printf '%s\n%s' "$a" "$b" | sort -V | tail -1)" == "$a" ]] +} + +# --------------------------------------------------------------------------- +# get_previous_version +# Clones community-operators-prod (reuses clone if present) and returns the +# highest version present in the given channel on stdout; all other output +# goes to stderr. +# --------------------------------------------------------------------------- +get_previous_version() { + local channel="$1" + local repo_path="$WORK_DIR/community-operators-prod" + + if [[ ! -d "$repo_path" ]]; then + echo "Cloning $COMMUNITY_OPERATORS_UPSTREAM" >&2 + git clone "$COMMUNITY_OPERATORS_UPSTREAM" "$repo_path" >&2 + fi + + git -C "$repo_path" checkout main >&2 + + local hive_dir="$repo_path/$HIVE_SUB_DIR" + local highest="0.0.0" + + while IFS= read -r version_dir; do + local version + version=$(basename "$version_dir") + + local annotation="$version_dir/metadata/annotations.yaml" + [[ -f "$annotation" ]] || continue + + local channels + channels=$(yq '.annotations["operators.operatorframework.io.bundle.channels.v1"] // ""' "$annotation") + + if [[ ",$channels," == *",$channel,"* ]]; then + if semver_gt "$version" "$highest"; then + highest="$version" + fi + fi + done < <(find "$hive_dir" -maxdepth 1 -mindepth 1 -type d | sort) + + if [[ "$highest" == "0.0.0" ]]; then + # NOTE: If a new channel is introduced, finding a prev_version will fail + # and require a manual edit of the CSV with the prev_version. The exit is + # intentionally fatal to ensure this failure is noticed rather than silently + # producing a broken bundle. + echo "Channel '$channel' not found in $COMMUNITY_OPERATORS_UPSTREAM" >&2 + exit 1 + fi + + echo "$highest" +} + +# --------------------------------------------------------------------------- +# generate_bundle +# Creates the full bundle directory structure under $bundle_dir/$semver_ver: +# metadata/annotations.yaml +# manifests/ +# manifests/hive-operator.v.clusterserviceversion.yaml +# +# Must be called from the hive repo root (reads config/ relative paths). +# --------------------------------------------------------------------------- +generate_bundle() { + local bundle_dir="$1" image_repo="$2" semver_ver="$3" image_tag="$4" + + local crds_dir="config/crds" + local csv_template="config/templates/hive-csv-template.yaml" + local operator_role="config/operator/operator_role.yaml" + local deployment_spec="config/operator/operator_deployment.yaml" + + local version_dir="$bundle_dir/$semver_ver" + local manifests_dir="$version_dir/manifests" + local metadata_dir="$version_dir/metadata" + mkdir -p "$manifests_dir" "$metadata_dir" + + echo "Writing bundle to: $version_dir" + echo "Generating CSV for version: $semver_ver" + + # --- metadata/annotations.yaml --- + cat > "$metadata_dir/annotations.yaml" < "$owned_crds_file" + + local crd_file + while IFS= read -r crd_file; do + cp "$crd_file" "$manifests_dir/" + + local kind version_name crd_name description + kind=$(yq '.spec.names.kind' "$crd_file") + version_name=$(yq '.spec.versions[0].name' "$crd_file") + crd_name=$(yq '.metadata.name' "$crd_file") + description=$(yq '.spec.versions[0].schema.openAPIV3Schema.description // ""' "$crd_file") + + KIND="$kind" VERSION="$version_name" CRD_NAME="$crd_name" DESCRIPTION="$description" \ + yq -i '. += [{"description": env(DESCRIPTION), "displayName": env(KIND), "kind": env(KIND), "name": env(CRD_NAME), "version": env(VERSION)}]' \ + "$owned_crds_file" + done < <(find "$crds_dir" -maxdepth 1 -type f \( -name '*.yaml' -o -name '*.yml' \) | sort) + + # --- Build CSV from template --- + local csv_file="$manifests_dir/hive-operator.v${semver_ver}.clusterserviceversion.yaml" + cp "$csv_template" "$csv_file" + + # Extract operator role rules and deployment spec into temp files so yq + # can load them as structured YAML (avoids quoting/escaping issues). + local rules_file="$WORK_DIR/rules.yaml" + local deploy_spec_file="$WORK_DIR/deploy-spec.yaml" + + yq '.rules' "$operator_role" > "$rules_file" + # operator_deployment.yaml is multi-document; index 1 is the Deployment. + yq 'select(document_index == 1) | .spec' "$deployment_spec" > "$deploy_spec_file" + + local image_ref="${image_repo}:${image_tag}" + local created_at + created_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + + yq -i " + .metadata.name = \"hive-operator.v${semver_ver}\" | + .spec.version = \"${semver_ver}\" | + .spec.customresourcedefinitions.owned = load(\"${owned_crds_file}\") | + .spec.install.spec.clusterPermissions = [{\"rules\": load(\"${rules_file}\"), \"serviceAccountName\": \"hive-operator\"}] | + .spec.install.spec.deployments[0].spec = load(\"${deploy_spec_file}\") | + .spec.install.spec.deployments[0].spec.template.spec.containers[0].image = \"${image_ref}\" | + .metadata.annotations.containerImage = \"${image_ref}\" | + .metadata.annotations.createdAt = \"${created_at}\" + " "$csv_file" + + echo "Wrote ClusterServiceVersion: $csv_file" +} + +# --------------------------------------------------------------------------- +# add_operatorhub_extras +# Adds OperatorHub-specific content to an already-generated bundle: +# - sets spec.replaces in the CSV +# - generates release-config.yaml (unless --skip-release-config) +# --------------------------------------------------------------------------- +add_operatorhub_extras() { + local version_dir="$1" semver_ver="$2" prev_version="$3" + + local csv_file="$version_dir/manifests/hive-operator.v${semver_ver}.clusterserviceversion.yaml" + yq -i ".spec.replaces = \"hive-operator.v${prev_version}\"" "$csv_file" + + if ! $SKIP_RELEASE_CONFIG; then + # release-config.yaml is only used by the Red Hat Ecosystem for automatic + # catalog updates across all RH catalogs. The Kubernetes Ecosystem ignores it. + cat > "$version_dir/release-config.yaml" < "org/repo" + local gh_target + gh_target=$(echo "$upstream_repo" | sed 's|git@github.com:||; s|\.git$||') + + if [[ ! -d "$repo_path" ]]; then + echo "Cloning $fork_repo" + git clone "$fork_repo" "$repo_path" + else + echo "Reusing existing clone at $repo_path" + fi + + git -C "$repo_path" remote set-url origin "$fork_repo" + git -C "$repo_path" remote add upstream "$upstream_repo" 2>/dev/null || \ + git -C "$repo_path" remote set-url upstream "$upstream_repo" + + echo "Fetching upstream $upstream_repo" + git -C "$repo_path" fetch upstream + + git -C "$repo_path" switch --detach upstream/main + + local branch_name="update-hive-${semver_ver}" + echo "Creating branch $branch_name" + git -C "$repo_path" switch -C "$branch_name" + + echo "Copying bundle" + cp -r "$bundle_dir/$semver_ver" "$repo_path/$HIVE_SUB_DIR/$semver_ver" + + local pr_title="operator hive-operator (${semver_ver})" + git -C "$repo_path" add "$HIVE_SUB_DIR" + git -C "$repo_path" commit --signoff --message="$pr_title" + + if $DRY_RUN; then + echo "DRY RUN: skipping push and PR for $gh_target" + return 0 + fi + + echo "Pushing branch $branch_name to origin" + git -C "$repo_path" push origin "$branch_name" --force + + local body="$pr_title" + $HOLD && body="${body}"$'\n\n/hold' + + echo "Opening PR to $gh_target" + gh pr create \ + --repo "$gh_target" \ + --head "${GITHUB_USER}:${branch_name}" \ + --base main \ + --title "$pr_title" \ + --body "$body" +} + +# --------------------------------------------------------------------------- +# check_deps — fail fast with a clear message if required tools are missing +# --------------------------------------------------------------------------- +check_deps() { + local missing=() + for cmd in yq jq git gh; do + command -v "$cmd" &>/dev/null || missing+=("$cmd") + done + if [[ ${#missing[@]} -gt 0 ]]; then + echo "Error: missing required tool(s): ${missing[*]}" >&2 + echo " yq (v4): https://github.com/mikefarah/yq" >&2 + echo " gh: https://cli.github.com/" >&2 + exit 1 + fi +} + +# --------------------------------------------------------------------------- +# main +# --------------------------------------------------------------------------- +main() { + parse_args "$@" + check_deps + $VERBOSE && set -x + + trap cleanup EXIT + + local orig_wd="$PWD" + + HIVE_REPO_DIR=$(mktemp -d --tmpdir hive-repo-XXXXXX) + BUNDLE_DIR=$(mktemp -d --tmpdir hive-operator-bundle-XXXXXX) + WORK_DIR=$(mktemp -d --tmpdir operatorhub-push-XXXXXX) + + echo "Cloning $HIVE_REPO to $HIVE_REPO_DIR" + git clone "$HIVE_REPO" "$HIVE_REPO_DIR" + + cd "$HIVE_REPO_DIR" + + # version2.sh sets globals: COMMIT, BRANCH_NAME, PREFIX, COMMIT_COUNT + # In OperatorHub mode the branch is always master; --dummy-bundle overrides + # this with an explicit branch (e.g. mce-2.1). Passing the branch explicitly + # avoids ambiguous auto-detection when the repo has many branches. + version_init "${DUMMY_BUNDLE:-master}" "${COMMIT_ISH:-}" + + local ver_semver image_tag + ver_semver=$(semver) + image_tag="${IMAGE_TAG_OVERRIDE:-${COMMIT:0:10}}" + + echo "Checking out $(shortsha)" + git checkout "$COMMIT" + + ensure_image "$IMAGE_REPO" "$image_tag" + + if [[ -n "$DUMMY_BUNDLE" ]]; then + # Mode 1: local bundle only — no version graph, no release-config, no PRs + generate_bundle "$BUNDLE_DIR" "$IMAGE_REPO" "$ver_semver" "$image_tag" + local dest="$orig_wd/hive-operator-bundle-v${ver_semver}" + # Remove any previous output for this version so cp -r replaces it + # cleanly rather than nesting the new bundle inside the existing dir. + rm -rf "$dest" + cp -r "$BUNDLE_DIR/$ver_semver" "$dest" + echo "Wrote bundle to $dest" + else + # Mode 2: OperatorHub push — bundle + graph directives + PRs + + # Enforce that OperatorHub bundles always use the master version prefix. + # This catches branch-detection failures early rather than letting a + # malformed version propagate into the community-operator repos. + if [[ "$PREFIX" != "$MASTER_BRANCH_PREFIX" ]]; then + echo "Error: version prefix '$PREFIX' does not match the expected master prefix '$MASTER_BRANCH_PREFIX'" >&2 + echo " The branch was likely not detected as master. Ensure --hive-repo points to the" >&2 + echo " upstream repo or a local clone checked out on master." >&2 + exit 1 + fi + + local prev_version + prev_version=$(get_previous_version "$CHANNEL_DEFAULT") + + if [[ "$ver_semver" == "$prev_version" ]]; then + echo "Error: version $ver_semver already exists upstream" >&2 + exit 1 + fi + if ! semver_gt "$ver_semver" "$prev_version"; then + echo "Error: version $ver_semver is lower than the current upstream version $prev_version" >&2 + echo " A newer bundle has already been published. Target a more recent commit on master." >&2 + exit 1 + fi + + generate_bundle "$BUNDLE_DIR" "$IMAGE_REPO" "$ver_semver" "$image_tag" + add_operatorhub_extras "$BUNDLE_DIR/$ver_semver" "$ver_semver" "$prev_version" + + open_pr \ + "git@github.com:${GITHUB_USER}/community-operators-prod.git" \ + "git@github.com:redhat-openshift-ecosystem/community-operators-prod.git" \ + "$ver_semver" "$BUNDLE_DIR" + + open_pr \ + "git@github.com:${GITHUB_USER}/community-operators.git" \ + "git@github.com:k8s-operatorhub/community-operators.git" \ + "$ver_semver" "$BUNDLE_DIR" + fi +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/hack/github.py b/hack/github.py deleted file mode 100644 index 6d33777b66f..00000000000 --- a/hack/github.py +++ /dev/null @@ -1,65 +0,0 @@ -import json -import os -import requests - - -class GitHubClient: - def __init__(self, user, repo, token=""): - gh_token = "" - - if token != "": - gh_token = token - else: - if os.environ.get("GITHUB_TOKEN") != None: - gh_token = os.environ["GITHUB_TOKEN"] - if os.environ.get("GH_TOKEN") != None: - gh_token = os.environ["GH_TOKEN"] - - if gh_token == "": - raise Exception("GitHub token not able to be set") - - self.user = user - self.repo = repo - self.headers = { - "Authorization": "token " + gh_token, - "Accept": "application/vnd.github.v3+json", - } - - def _create_request(self, rest_path): - return "https://api.github.com" + rest_path - - def create_annotated_tag(self, tag, tag_msg, commit_hash): - data = { - "tag": tag, - "message": tag_msg, - "object": commit_hash, - "type": "commit", - } - - create_tag_rest_path = "/repos/{}/{}/git/tags".format(self.user, self.repo) - req = self._create_request(create_tag_rest_path) - - return requests.post(req, headers=self.headers, data=json.dumps(data)) - - def create_reference(self, ref, sha): - data = { - "ref": ref, - "sha": sha, - } - - create_ref_rest_path = "/repos/{}/{}/git/refs".format(self.user, self.repo) - req = self._create_request(create_ref_rest_path) - - return requests.post(req, headers=self.headers, data=json.dumps(data)) - - def create_pr(self, pr_from, pr_to, pr_title, body): - data = { - "head": pr_from, - "base": pr_to, - "title": pr_title, - "body": body, - } - - create_pr_rest_path = "/repos/{}/{}/pulls".format(self.user, self.repo) - req = self._create_request(create_pr_rest_path) - return requests.post(req, headers=self.headers, data=json.dumps(data)) diff --git a/hack/requirements.txt b/hack/requirements.txt deleted file mode 100644 index 0e31bffdbea..00000000000 --- a/hack/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -GitPython -semver -PyYAML>=6.0 -requests diff --git a/hack/version2.sh b/hack/version2.sh index 1ae80e4ba6b..ca3f7726b2e 100755 --- a/hack/version2.sh +++ b/hack/version2.sh @@ -32,11 +32,7 @@ log() { shift local message="$*" - if [[ "$MODE" == "standalone" ]]; then - echo "$message" >&2 - else - echo "$message" - fi + echo "$message" >&2 if [[ "$level" == "fatal" ]]; then exit 1 @@ -143,9 +139,11 @@ validate_branch() { branch_from_commit() { local commit_ish="${1:-}" - local -a here=() - local -a ancestors=() - local -a descendants=() + # Use associative arrays so a branch appearing in different refs + # (refs/heads/, refs/remotes/origin/, etc.) is counted only once. + local -A here=() + local -A ancestors=() + local -A descendants=() # If we got an explicit commit_ish, and it's already a branch name, use it. if [[ -n "$commit_ish" ]]; then @@ -195,18 +193,18 @@ branch_from_commit() { # Okay, now start using $COMMIT, which is already rev_parse()d. # If this ref is at our commit, it's a candidate if [[ "$ref_commit" == "$COMMIT" ]]; then - here+=("$refname") + here["$refname"]=1 elif is_ancestor "$ref_commit" "$COMMIT"; then - ancestors+=("$refname") + ancestors["$refname"]=1 elif is_ancestor "$COMMIT" "$ref_commit"; then - descendants+=("$refname") + descendants["$refname"]=1 fi done < <(git for-each-ref --format='%(refname)' refs/heads/ refs/remotes/ 2>/dev/null) # First look for a unique branch right here if [[ ${#here[@]} -eq 1 ]]; then - log "debug" "Found unique branch ${here[0]} at commit $(shortsha)" - echo "${here[0]}" + log "debug" "Found unique branch ${!here[*]} at commit $(shortsha)" + echo "${!here[*]}" return 0 fi @@ -217,20 +215,20 @@ branch_from_commit() { # Next look for named descendants of our commit. This indicates we're versioning an old commit on the branch. if [[ ${#descendants[@]} -eq 1 ]]; then - log "debug" "Found branch ${descendants[0]} descended from commit $(shortsha)" - echo "${descendants[0]}" + log "debug" "Found branch ${!descendants[*]} descended from commit $(shortsha)" + echo "${!descendants[*]}" return 0 fi - log "debug" "descendants: ${descendants[*]}" + log "debug" "descendants: ${!descendants[*]}" if [[ ${#descendants[@]} -eq 0 ]]; then log "warning" "WARNING: Are you versioning an unmerged commit?" fi if [[ ${#ancestors[@]} -eq 1 ]]; then - log "debug" "Found branch ${ancestors[0]} which is an ancestor of commit $(shortsha)" - echo "${ancestors[0]}" + log "debug" "Found branch ${!ancestors[*]} which is an ancestor of commit $(shortsha)" + echo "${!ancestors[*]}" return 0 fi @@ -265,6 +263,10 @@ version_init() { if [[ "$(git rev-parse HEAD)" == "$COMMIT" ]]; then BRANCH_NAME="$active_branch" log "debug" "Using active branch $BRANCH_NAME since it corresponds to commit $(shortsha)" + else + # COMMIT differs from HEAD (e.g. an older commit was requested via + # --commit). Discover which branch this commit belongs to. + BRANCH_NAME=$(branch_from_commit) fi else # Detached HEAD: In CI builds (Konflux), always assume master since all