diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0034a4..3764134 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,59 +10,94 @@ on: jobs: build: - name: Continous Integration + name: Build & Test runs-on: ubuntu-latest steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v2 + - name: Check out code + uses: actions/checkout@v6 - - name: Set up Go 1.21.x - uses: actions/setup-go@v2 + - name: Set up Go + uses: actions/setup-go@v6 with: go-version: ^1.21 - id: go - name: Build - shell: bash - run: | - make build + run: go build -v ./... - name: Test - shell: bash - run: | - make test + run: go list ./... | grep -v /test | xargs -I% go test % -v -cover -race - scans: - name: Checks, Lints and Scans + lint: + name: Lint & Format runs-on: ubuntu-latest steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v2 + - name: Check out code + uses: actions/checkout@v6 - - name: Set up Go 1.21.x - uses: actions/setup-go@v2 + - name: Set up Go + uses: actions/setup-go@v6 with: go-version: ^1.21 - id: go - name: Format Check - shell: bash run: | - make format-check + RESULT=$(gofmt -l .) + if [ -n "$RESULT" ]; then + echo "Files need formatting:" + echo "$RESULT" + exit 1 + fi - name: Lint - shell: bash - run: | - make lint-docker + uses: golangci/golangci-lint-action@v9 + + security: + name: Security Scan + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v6 - name: Run Gosec Security Scanner - uses: securego/gosec@master + uses: securego/gosec@v2.22.11 with: - # let the report trigger content trigger a failure using the GitHub Security features. args: '-no-fail -fmt sarif -out results.sarif -exclude-dir internal -exclude-dir vendor -severity high ./...' - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@v1 + uses: github/codeql-action/upload-sarif@v4 with: - # path to SARIF file relative to the root of the repository sarif_file: results.sarif + + tag: + name: Auto Tag Release + needs: [build, lint, security] + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Extract version from CHANGELOG.md + id: version + run: | + VERSION=$(grep -oP '## \[\K[0-9]+\.[0-9]+\.[0-9]+' CHANGELOG.md | head -1) + echo "version=v${VERSION}" >> "$GITHUB_OUTPUT" + + - name: Check if tag already exists + id: check + run: | + if git rev-parse "${{ steps.version.outputs.version }}" >/dev/null 2>&1; then + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + - name: Create and push tag + if: steps.check.outputs.exists == 'false' + run: | + git tag "${{ steps.version.outputs.version }}" + git push origin "${{ steps.version.outputs.version }}" diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..e9b5268 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,11 @@ +version: "2" + +run: + timeout: 600s + +linters: + exclusions: + paths: + - test + - internal + - _mock.go diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9f6c638 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,15 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.1.0] - 2026-02-11 + +### Changed + +- Fork from vmware/vmware-go-kcl-v2 +- Update module path to github.com/ODudek/go-kcl +- Bump minimum Go version to 1.21 +- Upgrade AWS SDK and dependencies diff --git a/Makefile b/Makefile index bf3f0c5..a9368cc 100644 --- a/Makefile +++ b/Makefile @@ -3,10 +3,6 @@ help: ## - Show this help message @printf "\033[32m\xE2\x9c\x93 usage: make [target]\n\n\033[0m" @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -.PHONY: up -up: ## - start docker compose - @ cd _support/docker && docker-compose -f docker-compose.yml up - .PHONY: build-common build-common: ## - execute build common tasks clean and mod tidy @ go version @@ -22,15 +18,20 @@ build: build-common ## - build a debug binary to the current platform (windows, .PHONY: format-check format-check: ## - check files format using gofmt - @ ./_support/scripts/ci.sh fmtCheck - -.PHONY: format-check + @RESULT=$$(gofmt -l .); \ + if [ -n "$$RESULT" ]; then \ + echo "You need to run \"gofmt -w ./\" to fix your formatting."; \ + echo "$$RESULT"; \ + exit 1; \ + fi + +.PHONY: format format: ## - apply golang file format using gofmt - @ ./_support/scripts/ci.sh format + @ gofmt -w ./ .PHONY: test test: build-common ## - execute go test command for unit and mocked tests - @ ./_support/scripts/ci.sh unitTest + @ go list ./... | grep -v /test | xargs -I% go test % -v -cover -race .PHONY: integration-test integration-test: ## - execute go test command for integration tests (aws credentials needed) @@ -38,16 +39,8 @@ integration-test: ## - execute go test command for integration tests (aws creden .PHONY: scan scan: ## - execute static code analysis - @ ./_support/scripts/ci.sh scan - -.PHONY: local-scan -local-scan: ## - execute static code analysis locally - @ ./_support/scripts/ci.sh localScan + @ gosec -fmt=sarif -out=results.sarif -exclude-dir=internal -exclude-dir=vendor -severity=high ./... .PHONY: lint lint: ## - runs golangci-lint - @ ./_support/scripts/ci.sh lint - -.PHONY: lint-docker -lint-docker: ## - runs golangci-lint with docker container - @ ./_support/scripts/ci.sh lintDocker + @ golangci-lint run --timeout=600s --verbose diff --git a/_support/scripts/ci.sh b/_support/scripts/ci.sh deleted file mode 100755 index fb134fb..0000000 --- a/_support/scripts/ci.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash - -function local_go_pkgs() { - find './clientlibrary' -name '*.go' | \ - grep -Fv '/vendor/' | \ - grep -Fv '/go/' | \ - grep -Fv '/gen/' | \ - grep -Fv '/tmp/' | \ - grep -Fv '/run/' | \ - grep -Fv '/tests/' | \ - sed -r 's|(.+)/[^/]+\.go$|\1|g' | \ - sort -u -} - -function checkfmt() { - local files="" - files="$(find . -type f -iname "*.go" -exec gofmt -l {} \;)" - - if [ -n "$files" ]; then - echo "You need to run \"gofmt -w ./\" to fix your formatting." - echo "$files" >&2 - return 1 - fi -} - -function goFormat() { - echo "go formatting..." - gofmt -w ./ - echo "done" -} - -function lint() { - # golangci-lint run --enable-all -D forbidigo -D gochecknoglobals -D gofumpt -D gofmt -D nlreturn - - golangci-lint run \ - --skip-files=_mock.go \ - --skip-dirs=test \ - --skip-dirs=internal \ - --timeout=600s \ - --verbose -} - -function lintDocker() { - lintVersion="1.41.1" - lintImage="golangci/golangci-lint:v$lintVersion-alpine" - - docker run --rm -v "${PWD}":/app -w /app "$lintImage" golangci-lint run \ - --skip-files=_mock.go \ - --skip-dirs=test \ - --skip-dirs=internal \ - --timeout=600s \ - --verbose -} - -function unitTest() { - go list ./... | grep -v /test | \ - xargs -L 1 -I% bash -c 'echo -e "\n**************** Package: % ****************" && go test % -v -cover -race ./...' -} - -function scanast() { - gosec version - gosec ./... > security.log 2>&1 - - local issues="" - issues=$(grep -c 'Severity: MEDIUM' security.log | grep -v deaggregator | grep -c _) - if [ -n "$issues" ] && [ "$issues" -gt 0 ]; then - echo "" - echo "Medium Severity Issues:" - grep -e "Severity: MEDIUM" -A 1 security.log - echo "$issues" "medium severity issues found." - fi - - local issues="" - local issues_count="" - issues="$(grep -E 'Severity: HIGH' security.log | grep -v vendor)" - issues_count="$(grep -E 'Severity: HIGH' security.log | grep -v vendor | grep -c _)" - if [ -n "$issues_count" ] && [ "$issues_count" -gt 0 ]; then - echo "" - echo "High Severity Issues:" - grep -E "Severity: HIGH" -A 1 security.log - echo "$issues_count" "high severity issues found." - echo "$issues" - echo "You need to resolve the high severity issues at the least." - exit 1 - fi - - local issues="" - local issues_count="" - issues="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src)" - issues_count="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src | grep -c _)" - if [ -n "$issues_count" ] && [ "$issues_count" -gt 0 ]; then - echo "" - echo "Unhandled errors:" - grep -E "Errors unhandled" security.log - echo "$issues_count" "unhandled errors, please indicate with the right comment that this case is ok, or handle the error." - echo "$issues" - echo "You need to resolve the all unhandled errors." - exit 1 - fi - - rm -f security.log -} - -function scan() { - gosec -fmt=sarif -out=results.sarif -exclude-dir=internal -exclude-dir=vendor -severity=high ./... -} - -function localScan() { - # you can use the vs code plugin https://marketplace.visualstudio.com/items?itemName=MS-SarifVSCode.sarif-viewer - # to navigate against the issues - gosec -fmt=sarif -out=results.sarif -exclude-dir=internal -exclude-dir=vendor ./... -} - -function usage() { - echo "check.sh fmt|lint" >&2 - exit 2 -} - -case "$1" in - fmtCheck) checkfmt ;; - format) goFormat ;; - lint) lint ;; - lintDocker) lintDocker ;; - unitTest) unitTest ;; - scan) scan ;; - localScan) localScan ;; - *) usage ;; -esac diff --git a/clientlibrary/checkpoint/dynamodb-checkpointer.go b/clientlibrary/checkpoint/dynamodb-checkpointer.go index 7ba1749..3965059 100644 --- a/clientlibrary/checkpoint/dynamodb-checkpointer.go +++ b/clientlibrary/checkpoint/dynamodb-checkpointer.go @@ -52,7 +52,7 @@ const ( ) var ( - NoLeaseOwnerErr = errors.New("no LeaseOwner in checkpoints table") + ErrNoLeaseOwner = errors.New("no LeaseOwner in checkpoints table") ) // DynamoCheckpoint implements the Checkpoint interface using DynamoDB as a backend @@ -350,7 +350,7 @@ func (checkpointer *DynamoCheckpoint) GetLeaseOwner(shardID string) (string, err assignedVar, assignedToOk := currentCheckpoint[LeaseOwnerKey] if !assignedToOk { - return "", NoLeaseOwnerErr + return "", ErrNoLeaseOwner } return assignedVar.(*types.AttributeValueMemberS).Value, nil diff --git a/clientlibrary/worker/polling-shard-consumer.go b/clientlibrary/worker/polling-shard-consumer.go index d1b0350..ea34c9b 100644 --- a/clientlibrary/worker/polling-shard-consumer.go +++ b/clientlibrary/worker/polling-shard-consumer.go @@ -53,10 +53,10 @@ const ( ) var ( - rateLimitTimeNow = time.Now - rateLimitTimeSince = time.Since - localTPSExceededError = errors.New("Error GetRecords TPS Exceeded") - maxBytesExceededError = errors.New("Error GetRecords Max Bytes For Call Period Exceeded") + rateLimitTimeNow = time.Now + rateLimitTimeSince = time.Since + errLocalTPSExceeded = errors.New("error GetRecords TPS exceeded") + errMaxBytesExceeded = errors.New("error GetRecords max bytes for call period exceeded") ) // PollingShardConsumer is responsible for polling data records from a (specified) shard. @@ -175,13 +175,13 @@ func (sc *PollingShardConsumer) getRecords() error { sc.waitASecond(sc.currTime) continue } - if err == localTPSExceededError { - log.Infof("localTPSExceededError so sleep for a second") + if err == errLocalTPSExceeded { + log.Infof("errLocalTPSExceeded so sleep for a second") sc.waitASecond(sc.currTime) continue } - if err == maxBytesExceededError { - log.Infof("maxBytesExceededError so sleep for %+v seconds", coolDownPeriod) + if err == errMaxBytesExceeded { + log.Infof("errMaxBytesExceeded so sleep for %+v seconds", coolDownPeriod) time.Sleep(time.Duration(coolDownPeriod) * time.Second) continue } @@ -264,7 +264,7 @@ func (sc *PollingShardConsumer) checkCoolOffPeriod() (int, error) { if sc.bytesRead%MaxBytesPerSecond > 0 { coolDown++ } - return coolDown, maxBytesExceededError + return coolDown, errMaxBytesExceeded } else { sc.remBytes -= sc.bytesRead } @@ -285,7 +285,7 @@ func (sc *PollingShardConsumer) callGetRecordsAPI(gri *kinesis.GetRecordsInput) } if sc.callsLeft < 1 { - return nil, 0, localTPSExceededError + return nil, 0, errLocalTPSExceeded } getResp, err := sc.kc.GetRecords(context.TODO(), gri) sc.callsLeft-- diff --git a/clientlibrary/worker/polling-shard-consumer_test.go b/clientlibrary/worker/polling-shard-consumer_test.go index 736b2bd..bd4c693 100644 --- a/clientlibrary/worker/polling-shard-consumer_test.go +++ b/clientlibrary/worker/polling-shard-consumer_test.go @@ -51,7 +51,7 @@ func TestCallGetRecordsAPI(t *testing.T) { assert.Equal(t, &ret, out) m1.AssertExpectations(t) - // check that localTPSExceededError is thrown when trying more than 5 TPS + // check that errLocalTPSExceeded is thrown when trying more than 5 TPS m2 := MockKinesisSubscriberGetter{} psc2 := PollingShardConsumer{ commonShardConsumer: commonShardConsumer{kc: &m2}, @@ -62,7 +62,7 @@ func TestCallGetRecordsAPI(t *testing.T) { } out2, _, err2 := psc2.callGetRecordsAPI(&gri) assert.Nil(t, out2) - assert.ErrorIs(t, err2, localTPSExceededError) + assert.ErrorIs(t, err2, errLocalTPSExceeded) m2.AssertExpectations(t) // check that getRecords is called normally in bytesRead = 0 case