From 74f3cf5a65cd4fa4c9a4265fb1a8580434a90e20 Mon Sep 17 00:00:00 2001 From: Aleksander Date: Wed, 11 Feb 2026 03:37:17 +0100 Subject: [PATCH 1/4] Add CHANGELOG.md and CI auto-tag workflow for releases Introduce CHANGELOG.md following Keep a Changelog format. Update CI to auto-create and push version tags from CHANGELOG.md on main branch pushes. --- .github/workflows/ci.yml | 37 +++++++++++++++++++++++++++++++++++++ CHANGELOG.md | 15 +++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 CHANGELOG.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0034a4..be335b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,3 +66,40 @@ jobs: with: # path to SARIF file relative to the root of the repository sarif_file: results.sarif + + tag: + name: Auto Tag Release + needs: [build, scans] + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Check out code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Extract version from CHANGELOG.md + id: version + shell: bash + run: | + VERSION=$(grep -oP '## \[\K[0-9]+\.[0-9]+\.[0-9]+' CHANGELOG.md | head -1) + echo "version=v${VERSION}" >> "$GITHUB_OUTPUT" + + - name: Check if tag already exists + id: check + shell: bash + run: | + if git rev-parse "${{ steps.version.outputs.version }}" >/dev/null 2>&1; then + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + - name: Create and push tag + if: steps.check.outputs.exists == 'false' + shell: bash + run: | + git tag "${{ steps.version.outputs.version }}" + git push origin "${{ steps.version.outputs.version }}" diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9f6c638 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,15 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.1.0] - 2026-02-11 + +### Changed + +- Fork from vmware/vmware-go-kcl-v2 +- Update module path to github.com/ODudek/go-kcl +- Bump minimum Go version to 1.21 +- Upgrade AWS SDK and dependencies From 96632de7a527f05beae230d13f54987a9b5a49ef Mon Sep 17 00:00:00 2001 From: Aleksander Date: Wed, 11 Feb 2026 04:02:40 +0100 Subject: [PATCH 2/4] Refactor CI pipeline and Makefile for native Go tools - Simplify GitHub Actions workflows to use direct Go commands - Replace Makefile script wrappers with inline commands - Add .golangci.yml for consistent linting configuration - Remove obsolete helper scripts and Docker-based linting - Align build, test, lint, and scan steps with project conventions --- .github/workflows/ci.yml | 64 ++++++++++---------- .golangci.yml | 9 +++ Makefile | 31 ++++------ _support/scripts/ci.sh | 128 --------------------------------------- 4 files changed, 52 insertions(+), 180 deletions(-) create mode 100644 .golangci.yml delete mode 100755 _support/scripts/ci.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be335b2..50ed497 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,86 +10,85 @@ on: jobs: build: - name: Continous Integration + name: Build & Test runs-on: ubuntu-latest steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v2 + - name: Check out code + uses: actions/checkout@v6 - - name: Set up Go 1.21.x - uses: actions/setup-go@v2 + - name: Set up Go + uses: actions/setup-go@v6 with: go-version: ^1.21 - id: go - name: Build - shell: bash - run: | - make build + run: go build -v ./... - name: Test - shell: bash - run: | - make test + run: go list ./... | grep -v /test | xargs -I% go test % -v -cover -race - scans: - name: Checks, Lints and Scans + lint: + name: Lint & Format runs-on: ubuntu-latest steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v2 + - name: Check out code + uses: actions/checkout@v6 - - name: Set up Go 1.21.x - uses: actions/setup-go@v2 + - name: Set up Go + uses: actions/setup-go@v6 with: go-version: ^1.21 - id: go - name: Format Check - shell: bash run: | - make format-check + RESULT=$(gofmt -l .) + if [ -n "$RESULT" ]; then + echo "Files need formatting:" + echo "$RESULT" + exit 1 + fi - name: Lint - shell: bash - run: | - make lint-docker + uses: golangci/golangci-lint-action@v9 + + security: + name: Security Scan + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v6 - name: Run Gosec Security Scanner - uses: securego/gosec@master + uses: securego/gosec@v2 with: - # let the report trigger content trigger a failure using the GitHub Security features. args: '-no-fail -fmt sarif -out results.sarif -exclude-dir internal -exclude-dir vendor -severity high ./...' - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@v1 + uses: github/codeql-action/upload-sarif@v4 with: - # path to SARIF file relative to the root of the repository sarif_file: results.sarif tag: name: Auto Tag Release - needs: [build, scans] + needs: [build, lint, security] if: github.ref == 'refs/heads/main' && github.event_name == 'push' runs-on: ubuntu-latest permissions: contents: write steps: - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v6 with: fetch-depth: 0 - name: Extract version from CHANGELOG.md id: version - shell: bash run: | VERSION=$(grep -oP '## \[\K[0-9]+\.[0-9]+\.[0-9]+' CHANGELOG.md | head -1) echo "version=v${VERSION}" >> "$GITHUB_OUTPUT" - name: Check if tag already exists id: check - shell: bash run: | if git rev-parse "${{ steps.version.outputs.version }}" >/dev/null 2>&1; then echo "exists=true" >> "$GITHUB_OUTPUT" @@ -99,7 +98,6 @@ jobs: - name: Create and push tag if: steps.check.outputs.exists == 'false' - shell: bash run: | git tag "${{ steps.version.outputs.version }}" git push origin "${{ steps.version.outputs.version }}" diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..f54e6ca --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,9 @@ +run: + timeout: 600s + +issues: + exclude-dirs: + - test + - internal + exclude-files: + - _mock.go diff --git a/Makefile b/Makefile index bf3f0c5..a9368cc 100644 --- a/Makefile +++ b/Makefile @@ -3,10 +3,6 @@ help: ## - Show this help message @printf "\033[32m\xE2\x9c\x93 usage: make [target]\n\n\033[0m" @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -.PHONY: up -up: ## - start docker compose - @ cd _support/docker && docker-compose -f docker-compose.yml up - .PHONY: build-common build-common: ## - execute build common tasks clean and mod tidy @ go version @@ -22,15 +18,20 @@ build: build-common ## - build a debug binary to the current platform (windows, .PHONY: format-check format-check: ## - check files format using gofmt - @ ./_support/scripts/ci.sh fmtCheck - -.PHONY: format-check + @RESULT=$$(gofmt -l .); \ + if [ -n "$$RESULT" ]; then \ + echo "You need to run \"gofmt -w ./\" to fix your formatting."; \ + echo "$$RESULT"; \ + exit 1; \ + fi + +.PHONY: format format: ## - apply golang file format using gofmt - @ ./_support/scripts/ci.sh format + @ gofmt -w ./ .PHONY: test test: build-common ## - execute go test command for unit and mocked tests - @ ./_support/scripts/ci.sh unitTest + @ go list ./... | grep -v /test | xargs -I% go test % -v -cover -race .PHONY: integration-test integration-test: ## - execute go test command for integration tests (aws credentials needed) @@ -38,16 +39,8 @@ integration-test: ## - execute go test command for integration tests (aws creden .PHONY: scan scan: ## - execute static code analysis - @ ./_support/scripts/ci.sh scan - -.PHONY: local-scan -local-scan: ## - execute static code analysis locally - @ ./_support/scripts/ci.sh localScan + @ gosec -fmt=sarif -out=results.sarif -exclude-dir=internal -exclude-dir=vendor -severity=high ./... .PHONY: lint lint: ## - runs golangci-lint - @ ./_support/scripts/ci.sh lint - -.PHONY: lint-docker -lint-docker: ## - runs golangci-lint with docker container - @ ./_support/scripts/ci.sh lintDocker + @ golangci-lint run --timeout=600s --verbose diff --git a/_support/scripts/ci.sh b/_support/scripts/ci.sh deleted file mode 100755 index fb134fb..0000000 --- a/_support/scripts/ci.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash - -function local_go_pkgs() { - find './clientlibrary' -name '*.go' | \ - grep -Fv '/vendor/' | \ - grep -Fv '/go/' | \ - grep -Fv '/gen/' | \ - grep -Fv '/tmp/' | \ - grep -Fv '/run/' | \ - grep -Fv '/tests/' | \ - sed -r 's|(.+)/[^/]+\.go$|\1|g' | \ - sort -u -} - -function checkfmt() { - local files="" - files="$(find . -type f -iname "*.go" -exec gofmt -l {} \;)" - - if [ -n "$files" ]; then - echo "You need to run \"gofmt -w ./\" to fix your formatting." - echo "$files" >&2 - return 1 - fi -} - -function goFormat() { - echo "go formatting..." - gofmt -w ./ - echo "done" -} - -function lint() { - # golangci-lint run --enable-all -D forbidigo -D gochecknoglobals -D gofumpt -D gofmt -D nlreturn - - golangci-lint run \ - --skip-files=_mock.go \ - --skip-dirs=test \ - --skip-dirs=internal \ - --timeout=600s \ - --verbose -} - -function lintDocker() { - lintVersion="1.41.1" - lintImage="golangci/golangci-lint:v$lintVersion-alpine" - - docker run --rm -v "${PWD}":/app -w /app "$lintImage" golangci-lint run \ - --skip-files=_mock.go \ - --skip-dirs=test \ - --skip-dirs=internal \ - --timeout=600s \ - --verbose -} - -function unitTest() { - go list ./... | grep -v /test | \ - xargs -L 1 -I% bash -c 'echo -e "\n**************** Package: % ****************" && go test % -v -cover -race ./...' -} - -function scanast() { - gosec version - gosec ./... > security.log 2>&1 - - local issues="" - issues=$(grep -c 'Severity: MEDIUM' security.log | grep -v deaggregator | grep -c _) - if [ -n "$issues" ] && [ "$issues" -gt 0 ]; then - echo "" - echo "Medium Severity Issues:" - grep -e "Severity: MEDIUM" -A 1 security.log - echo "$issues" "medium severity issues found." - fi - - local issues="" - local issues_count="" - issues="$(grep -E 'Severity: HIGH' security.log | grep -v vendor)" - issues_count="$(grep -E 'Severity: HIGH' security.log | grep -v vendor | grep -c _)" - if [ -n "$issues_count" ] && [ "$issues_count" -gt 0 ]; then - echo "" - echo "High Severity Issues:" - grep -E "Severity: HIGH" -A 1 security.log - echo "$issues_count" "high severity issues found." - echo "$issues" - echo "You need to resolve the high severity issues at the least." - exit 1 - fi - - local issues="" - local issues_count="" - issues="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src)" - issues_count="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src | grep -c _)" - if [ -n "$issues_count" ] && [ "$issues_count" -gt 0 ]; then - echo "" - echo "Unhandled errors:" - grep -E "Errors unhandled" security.log - echo "$issues_count" "unhandled errors, please indicate with the right comment that this case is ok, or handle the error." - echo "$issues" - echo "You need to resolve the all unhandled errors." - exit 1 - fi - - rm -f security.log -} - -function scan() { - gosec -fmt=sarif -out=results.sarif -exclude-dir=internal -exclude-dir=vendor -severity=high ./... -} - -function localScan() { - # you can use the vs code plugin https://marketplace.visualstudio.com/items?itemName=MS-SarifVSCode.sarif-viewer - # to navigate against the issues - gosec -fmt=sarif -out=results.sarif -exclude-dir=internal -exclude-dir=vendor ./... -} - -function usage() { - echo "check.sh fmt|lint" >&2 - exit 2 -} - -case "$1" in - fmtCheck) checkfmt ;; - format) goFormat ;; - lint) lint ;; - lintDocker) lintDocker ;; - unitTest) unitTest ;; - scan) scan ;; - localScan) localScan ;; - *) usage ;; -esac From a8188ea4b517dff43746bd911cb892b896ef5abb Mon Sep 17 00:00:00 2001 From: Aleksander Date: Wed, 11 Feb 2026 04:11:03 +0100 Subject: [PATCH 3/4] Refactor error variable names for consistency and clarity - Rename error vars in polling-shard-consumer and dynamodb-checkpointer - Update related log messages and tests to match new names --- .github/workflows/ci.yml | 2 +- .../checkpoint/dynamodb-checkpointer.go | 4 ++-- .../worker/polling-shard-consumer.go | 20 +++++++++---------- .../worker/polling-shard-consumer_test.go | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50ed497..3764134 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: uses: actions/checkout@v6 - name: Run Gosec Security Scanner - uses: securego/gosec@v2 + uses: securego/gosec@v2.22.11 with: args: '-no-fail -fmt sarif -out results.sarif -exclude-dir internal -exclude-dir vendor -severity high ./...' diff --git a/clientlibrary/checkpoint/dynamodb-checkpointer.go b/clientlibrary/checkpoint/dynamodb-checkpointer.go index 7ba1749..3965059 100644 --- a/clientlibrary/checkpoint/dynamodb-checkpointer.go +++ b/clientlibrary/checkpoint/dynamodb-checkpointer.go @@ -52,7 +52,7 @@ const ( ) var ( - NoLeaseOwnerErr = errors.New("no LeaseOwner in checkpoints table") + ErrNoLeaseOwner = errors.New("no LeaseOwner in checkpoints table") ) // DynamoCheckpoint implements the Checkpoint interface using DynamoDB as a backend @@ -350,7 +350,7 @@ func (checkpointer *DynamoCheckpoint) GetLeaseOwner(shardID string) (string, err assignedVar, assignedToOk := currentCheckpoint[LeaseOwnerKey] if !assignedToOk { - return "", NoLeaseOwnerErr + return "", ErrNoLeaseOwner } return assignedVar.(*types.AttributeValueMemberS).Value, nil diff --git a/clientlibrary/worker/polling-shard-consumer.go b/clientlibrary/worker/polling-shard-consumer.go index d1b0350..ea34c9b 100644 --- a/clientlibrary/worker/polling-shard-consumer.go +++ b/clientlibrary/worker/polling-shard-consumer.go @@ -53,10 +53,10 @@ const ( ) var ( - rateLimitTimeNow = time.Now - rateLimitTimeSince = time.Since - localTPSExceededError = errors.New("Error GetRecords TPS Exceeded") - maxBytesExceededError = errors.New("Error GetRecords Max Bytes For Call Period Exceeded") + rateLimitTimeNow = time.Now + rateLimitTimeSince = time.Since + errLocalTPSExceeded = errors.New("error GetRecords TPS exceeded") + errMaxBytesExceeded = errors.New("error GetRecords max bytes for call period exceeded") ) // PollingShardConsumer is responsible for polling data records from a (specified) shard. @@ -175,13 +175,13 @@ func (sc *PollingShardConsumer) getRecords() error { sc.waitASecond(sc.currTime) continue } - if err == localTPSExceededError { - log.Infof("localTPSExceededError so sleep for a second") + if err == errLocalTPSExceeded { + log.Infof("errLocalTPSExceeded so sleep for a second") sc.waitASecond(sc.currTime) continue } - if err == maxBytesExceededError { - log.Infof("maxBytesExceededError so sleep for %+v seconds", coolDownPeriod) + if err == errMaxBytesExceeded { + log.Infof("errMaxBytesExceeded so sleep for %+v seconds", coolDownPeriod) time.Sleep(time.Duration(coolDownPeriod) * time.Second) continue } @@ -264,7 +264,7 @@ func (sc *PollingShardConsumer) checkCoolOffPeriod() (int, error) { if sc.bytesRead%MaxBytesPerSecond > 0 { coolDown++ } - return coolDown, maxBytesExceededError + return coolDown, errMaxBytesExceeded } else { sc.remBytes -= sc.bytesRead } @@ -285,7 +285,7 @@ func (sc *PollingShardConsumer) callGetRecordsAPI(gri *kinesis.GetRecordsInput) } if sc.callsLeft < 1 { - return nil, 0, localTPSExceededError + return nil, 0, errLocalTPSExceeded } getResp, err := sc.kc.GetRecords(context.TODO(), gri) sc.callsLeft-- diff --git a/clientlibrary/worker/polling-shard-consumer_test.go b/clientlibrary/worker/polling-shard-consumer_test.go index 736b2bd..bd4c693 100644 --- a/clientlibrary/worker/polling-shard-consumer_test.go +++ b/clientlibrary/worker/polling-shard-consumer_test.go @@ -51,7 +51,7 @@ func TestCallGetRecordsAPI(t *testing.T) { assert.Equal(t, &ret, out) m1.AssertExpectations(t) - // check that localTPSExceededError is thrown when trying more than 5 TPS + // check that errLocalTPSExceeded is thrown when trying more than 5 TPS m2 := MockKinesisSubscriberGetter{} psc2 := PollingShardConsumer{ commonShardConsumer: commonShardConsumer{kc: &m2}, @@ -62,7 +62,7 @@ func TestCallGetRecordsAPI(t *testing.T) { } out2, _, err2 := psc2.callGetRecordsAPI(&gri) assert.Nil(t, out2) - assert.ErrorIs(t, err2, localTPSExceededError) + assert.ErrorIs(t, err2, errLocalTPSExceeded) m2.AssertExpectations(t) // check that getRecords is called normally in bytesRead = 0 case From 938bf05d43a099a9ce2289063bc7f0b34de2ff15 Mon Sep 17 00:00:00 2001 From: Aleksander Date: Wed, 11 Feb 2026 04:11:14 +0100 Subject: [PATCH 4/4] Update .golangci.yml --- .golangci.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index f54e6ca..e9b5268 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,9 +1,11 @@ +version: "2" + run: timeout: 600s -issues: - exclude-dirs: - - test - - internal - exclude-files: - - _mock.go +linters: + exclusions: + paths: + - test + - internal + - _mock.go