Build

wip: test AI generated PR reviews #446

Workflow file for this run

	name: Build

	on:
	push:
	pull_request:
	types: [opened, synchronize, reopened]
	workflow_dispatch:

	env:
	PACT_BROKER_BASE_URL: ${{ secrets.PACT_BROKER_BASE_URL }}
	PACT_BROKER_TOKEN: ${{ secrets.PACTFLOW_TOKEN_FOR_CI_CD_WORKSHOP }}
	REACT_APP_API_BASE_URL: http://localhost:3001
	GIT_COMMIT: ${{ github.sha }}
	GIT_REF: ${{ github.ref }}

	jobs:
	test:
	runs-on: ubuntu-latest
	strategy:
	matrix:
	pact_provider: [
	"pactflow-example-bi-directional-provider-dredd",
	# "pactflow-example-bi-directional-provider-restassured",
	# "pactflow-example-bi-directional-provider-postman",
	'pactflow-example-provider'
	]
	steps:
	- uses: actions/checkout@v3
	- uses: actions/setup-node@v3
	with:
	node-version: '18'
	- name: Install
	run: npm i
	- name: Test
	env:
	PACT_PROVIDER: ${{ matrix.pact_provider }}
	run: make test
	- name: Publish pacts between pactflow-example-consumer and ${{ matrix.pact_provider }}
	run: GIT_BRANCH=${GIT_REF:11} make publish_pacts
	env:
	PACT_PROVIDER: ${{ matrix.pact_provider }}

	# Runs on branches as well, so we know the status of our PRs
	can-i-deploy:
	runs-on: ubuntu-latest
	needs: test
	steps:
	- uses: actions/checkout@v3
	- run: docker pull pactfoundation/pact-cli:latest
	- name: Can I deploy?
	run: GIT_BRANCH=${GIT_REF:11} make can_i_deploy

	# Claude AI powered Pact test coverage review for PRs
	pact-coverage-review:
	runs-on: ubuntu-latest
	if: github.event_name == 'pull_request'
	permissions:
	contents: read
	pull-requests: write
	steps:
	- uses: actions/checkout@v3
	with:
	fetch-depth: 0 # Need full history for diff analysis

	- uses: actions/setup-node@v3
	with:
	node-version: '18'

	- name: Install dependencies
	run: npm ci

	- name: Install Claude CLI
	run: \|
	curl -fsSL https://claude.ai/cli/install.sh \| sh
	echo "$HOME/.local/bin" >> $GITHUB_PATH
	env:
	CLAUDE_API_KEY: ${{ secrets.CLAUDE_API_KEY }}

	- name: Setup SmartBear MCP Server
	run: \|
	# Install SmartBear MCP server globally
	npm install -g @smartbear/mcp@latest

	# Create MCP config for Claude CLI
	mkdir -p ~/.config/claude
	cat > ~/.config/claude/mcp.json << 'EOF'
	{
	"servers": {
	"smartbear": {
	"type": "stdio",
	"command": "npx",
	"args": ["-y", "@smartbear/mcp@latest"],
	"env": {
	"PACT_BROKER_BASE_URL": "${{ secrets.PACT_BROKER_BASE_URL }}",
	"PACT_BROKER_TOKEN": "${{ secrets.PACTFLOW_TOKEN_FOR_CI_CD_WORKSHOP }}"
	}
	}
	}
	}
	EOF

	- name: Analyze API Client Coverage
	id: coverage-analysis
	run: \|
	# Use the dedicated coverage analysis script
	node .github/scripts/analyze-pact-coverage.js

	# Output key metrics for use in later steps
	if [ -f coverage-report.json ]; then
	COVERAGE_PERCENT=$(node -e "console.log(JSON.parse(require('fs').readFileSync('coverage-report.json', 'utf8')).summary.coveragePercentage)")
	TOTAL_METHODS=$(node -e "console.log(JSON.parse(require('fs').readFileSync('coverage-report.json', 'utf8')).summary.totalApiMethods)")
	COVERED_METHODS=$(node -e "console.log(JSON.parse(require('fs').readFileSync('coverage-report.json', 'utf8')).summary.coveredMethods)")

	echo "coverage_percent=$COVERAGE_PERCENT" >> $GITHUB_OUTPUT
	echo "total_methods=$TOTAL_METHODS" >> $GITHUB_OUTPUT
	echo "covered_methods=$COVERED_METHODS" >> $GITHUB_OUTPUT

	echo "📊 Coverage Analysis Complete:"
	echo " - Total API Methods: $TOTAL_METHODS"
	echo " - Covered Methods: $COVERED_METHODS"
	echo " - Coverage: $COVERAGE_PERCENT%"
	else
	echo "❌ Coverage analysis failed"
	echo "coverage_percent=0" >> $GITHUB_OUTPUT
	echo "total_methods=0" >> $GITHUB_OUTPUT
	echo "covered_methods=0" >> $GITHUB_OUTPUT
	fi

	- name: Generate Coverage Report and Recommendations
	env:
	ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
	run: \|
	# Create comprehensive Claude prompt that utilizes SmartBear MCP tools
	cat > claude_analysis_prompt.txt << 'EOF'
	You are an expert in API testing and Pact contract testing. Analyze this JavaScript project's API client and Pact test coverage comprehensively, then generate complete code to achieve 100% coverage.

	## Context
	- Project: Node.js consumer application using Pact for contract testing
	- API Client: `src/api.js` - Contains methods that call a Product API
	- Pact Tests: `src/api.pact.spec.ts` - Should contain contract tests (currently mostly empty)
	- Domain Model: `src/product.js` - Product class definition
	- OpenAPI Spec: `products.yml` - API specification
	- Test Template: `src/pact.test.template` - Template for writing Pact tests
	- Instructions: `src/test.instructions.txt` - Specific testing guidelines
	- Existing Pact: `pacts/ProductConsumer-ProductProvider.json` - Current contract

	## Analysis Required
	1. Coverage Analysis: Examine the generated coverage report and identify gaps
	2. API Client Review: Check all methods in the API client for proper Pact test coverage
	3. Scenario Coverage: Verify coverage of HTTP status codes (200, 400, 401, 404)
	4. OpenAPI Alignment: Check if API client and tests align with OpenAPI specification
	5. Test Quality: Review existing Pact interactions for completeness and best practices

	## Code Generation Task
	PRIMARY GOAL: Use SmartBear MCP tools to generate a complete, working Pact test file that achieves 100% coverage.

	Required Actions:
	1. Call `mcp__smartbear__contract-testing__generate_pact_tests` for EACH API method in src/api.js
	2. Generate comprehensive test scenarios for each endpoint:
	- Success scenarios (200)
	- Authentication failures (401)
	- Not found errors (404)
	- Bad request errors (400) where applicable
	3. Ensure all tests follow the project's template and instructions
	4. Generate a complete replacement for src/api.pact.spec.ts

	SmartBear MCP Tool Parameters:
	For each API method, use these parameters:
	```json
	{
	"language": "typescript",
	"code": [
	{"filename": "api.js", "body": "[full API client code]", "language": "javascript"},
	{"filename": "product.js", "body": "[Product model code]", "language": "javascript"}
	],
	"openapi": {
	"document": "[complete OpenAPI spec]",
	"matcher": {
	"path": "[endpoint path like /products or /product/{id}]",
	"methods": ["GET", "DELETE"],
	"statusCodes": [200, 400, 401, 404],
	"operationId": "[operation ID from OpenAPI]"
	}
	},
	"testTemplate": {"filename": "pact.test.template", "body": "[template content]"},
	"additionalInstructions": "Follow src/test.instructions.txt exactly. Cover happy and non-happy paths (200, 400, 401, 404). Use Jest with async/await. Use PactV4 interface. Include proper Authorization headers using Bearer tokens."
	}
	```

	## Expected Output Format
	Structure your analysis as a comprehensive markdown report with ACTUAL GENERATED CODE:

	```markdown
	# 🤖 Pact Coverage Analysis & Code Generation

	## 📊 Current Coverage Status
	[Include coverage statistics and analysis]

	## 🔍 Detailed Analysis
	[Method-by-method breakdown with gaps identified]

	## � Generated Pact Test Code
	[Complete, runnable TypeScript code generated by SmartBear MCP tools]

	## � Critical Issues Found
	[List any critical problems and how the generated code addresses them]

	## ✅ Implementation Instructions
	[Step-by-step guide to implement the generated code]
	```

	Important:
	- MUST use SmartBear MCP tools to generate actual, working test code
	- The generated code should be complete and ready to replace src/api.pact.spec.ts
	- Include all necessary imports, setup, and test cases
	- Follow the project's existing patterns and style exactly
	- Generate tests that will achieve 100% API client coverage
	EOF

	# Execute comprehensive analysis with Claude + SmartBear MCP
	echo "🔍 Running Claude analysis with SmartBear MCP tools..."

	claude \
	--mcp-config ~/.config/claude/mcp.json \
	--allowedTools "Read,Write,Bash,mcp__smartbear__contract-testing__generate_pact_tests,mcp__smartbear__contract-testing__review_pact_tests,mcp__smartbear__contract-testing__get_provider_states" \
	--permission-mode acceptEdits \
	--prompt @claude_analysis_prompt.txt \
	--files coverage-report.json,coverage-analysis.md,src/api.js,src/api.pact.spec.ts,src/product.js,products.yml,pacts/ProductConsumer-ProductProvider.json,src/pact.test.template,src/test.instructions.txt \
	--output comprehensive-analysis.md \
	--max-tokens 4000

	# Also generate specific test code using SmartBear MCP if coverage is low
	COVERAGE_PERCENT="${{ steps.coverage-analysis.outputs.coverage_percent }}"
	if [ "$COVERAGE_PERCENT" -lt 80 ]; then
	echo "🔧 Coverage below 80%, generating specific test recommendations..."

	cat > test_generation_prompt.txt << 'EOF'
	The Pact test coverage is below 80%. Generate a complete, working Pact test file to achieve 100% coverage.

	Objective: Create a comprehensive src/api.pact.spec.ts file that covers ALL API client methods.

	Requirements:
	1. Complete Coverage: Test every method in src/api.js
	2. All Scenarios: Include 200, 400, 401, and 404 test cases for each endpoint
	3. Follow Template: Use the exact structure from src/pact.test.template
	4. Follow Instructions: Adhere strictly to src/test.instructions.txt
	5. Working Code: Generate runnable TypeScript that passes tests

	SmartBear MCP Tool Usage:
	Call `mcp__smartbear__contract-testing__generate_pact_tests` for each API endpoint:

	1. getAllProducts() - GET /products
	2. getProduct(id) - GET /product/{id}
	3. deleteProduct(id) - DELETE /product/{id}

	Parameters for each call:
	```json
	{
	"language": "typescript",
	"code": [
	{"filename": "api.js", "body": "[API client code]", "language": "javascript"},
	{"filename": "product.js", "body": "[Product model code]", "language": "javascript"}
	],
	"openapi": {
	"document": "[OpenAPI specification]",
	"matcher": {
	"path": "[endpoint path]",
	"methods": ["GET", "DELETE"],
	"statusCodes": [200, 400, 401, 404],
	"operationId": "[operation ID]"
	}
	},
	"testTemplate": {"filename": "pact.test.template", "body": "[template]"},
	"additionalInstructions": "Follow test.instructions.txt exactly. Cover happy and non-happy paths (200, 400, 401, 404). Use Jest framework with async/await. Use PactV4 interface. Include proper authorization headers. Omit template comments."
	}
	```

	Output: Provide the complete content for src/api.pact.spec.ts that achieves 100% API client coverage.
	EOF

	claude \
	--mcp-config ~/.config/claude/mcp.json \
	--allowedTools "Read,Write,Bash,mcp__smartbear__contract-testing__generate_pact_tests,mcp__smartbear__contract-testing__review_pact_tests" \
	--permission-mode acceptEdits \
	--prompt @test_generation_prompt.txt \
	--files src/api.js,src/product.js,products.yml,src/pact.test.template,src/test.instructions.txt \
	--output generated-pact-tests.ts \
	--max-tokens 3000
	fi

	- name: Check for API Client Changes in PR
	id: check-changes
	run: \|
	# Check if PR contains changes to API client or Pact tests
	git diff origin/${{ github.base_ref }}...HEAD --name-only > changed_files.txt

	API_CHANGES=false
	PACT_CHANGES=false

	if grep -q "src/api\.js\\|src/.*\.pact\.spec\.ts\\|products\.yml" changed_files.txt; then
	API_CHANGES=true
	fi

	if grep -q "src/.*\.pact\.spec\.ts\\|pacts/" changed_files.txt; then
	PACT_CHANGES=true
	fi

	echo "api_changes=$API_CHANGES" >> $GITHUB_OUTPUT
	echo "pact_changes=$PACT_CHANGES" >> $GITHUB_OUTPUT
	echo "changed_files=$(cat changed_files.txt \| tr '\n' ',' \| sed 's/,$//')" >> $GITHUB_OUTPUT

	- name: Generate Specific PR Recommendations
	if: steps.check-changes.outputs.api_changes == 'true'
	env:
	ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
	run: \|
	# Create specific analysis for PR changes
	cat > pr_analysis_prompt.txt << 'EOF'
	Analyze the specific changes in this PR that affect the API client and/or Pact tests, then generate concrete code to achieve 100% test coverage.

	Changed Files: ${{ steps.check-changes.outputs.changed_files }}

	Primary Tasks:
	1. Change Analysis: Review the specific changes made to API client code
	2. Gap Identification: Identify if new API methods were added or existing ones modified
	3. Coverage Assessment: Check if corresponding Pact tests exist for the changes
	4. Code Generation: Use SmartBear MCP tools to generate complete, runnable Pact tests
	5. Specification Alignment: Verify the changes align with the OpenAPI specification

	Code Generation Requirements:
	- Use `mcp__smartbear__contract-testing__generate_pact_tests` to create comprehensive Pact tests
	- Generate tests for EVERY API method that lacks coverage
	- Include ALL scenarios: 200 (success), 400 (bad request), 401 (unauthorized), 404 (not found)
	- Follow the project's test template and instructions exactly
	- Generate complete, runnable TypeScript code that can replace src/api.pact.spec.ts

	Expected Output:
	1. Analysis Summary: What changed and what needs testing
	2. Coverage Gaps: Specific missing tests identified
	3. Generated Code: Complete Pact test file content using SmartBear MCP tools
	4. Implementation Guide: Step-by-step instructions for the developer

	SmartBear MCP Tool Usage:
	For each API method lacking coverage, call `mcp__smartbear__contract-testing__generate_pact_tests` with:
	```json
	{
	"language": "typescript",
	"code": [
	{"filename": "api.js", "body": "[API client code]"},
	{"filename": "product.js", "body": "[Product model code]"}
	],
	"openapi": {
	"document": "[OpenAPI spec]",
	"matcher": {
	"path": "[specific endpoint path]",
	"methods": ["GET", "POST", "DELETE"],
	"statusCodes": [200, 400, 401, 404],
	"operationId": "[operation ID]"
	}
	},
	"testTemplate": {"body": "[Pact test template]"},
	"additionalInstructions": "Follow test.instructions.txt exactly. Cover all HTTP status codes. Use Jest with async/await. Include proper authorization headers."
	}
	```

	Goal: Generate a complete, working Pact test suite that achieves 100% coverage of the API client.
	EOF

	# Get the diff for changed files
	git diff origin/${{ github.base_ref }}...HEAD -- src/api.js src/*.pact.spec.ts products.yml > pr_changes.diff

	# Run targeted analysis with full tool access
	claude \
	--mcp-config ~/.config/claude/mcp.json \
	--allowedTools "Read,Write,Bash,mcp__smartbear__contract-testing__generate_pact_tests,mcp__smartbear__contract-testing__review_pact_tests,mcp__smartbear__contract-testing__get_provider_states" \
	--permission-mode acceptEdits \
	--prompt @pr_analysis_prompt.txt \
	--files pr_changes.diff,coverage-analysis.md,src/api.js,src/product.js,products.yml,src/pact.test.template,src/test.instructions.txt \
	--output pr-recommendations.md \
	--max-tokens 4000

	- name: Comment on PR with Analysis
	uses: actions/github-script@v7
	with:
	script: \|
	const fs = require('fs');

	// Read all generated analyses
	let baseAnalysis = '';
	let comprehensiveAnalysis = '';
	let prRecommendations = '';
	let generatedTests = '';
	let coverageStats = '';

	try {
	baseAnalysis = fs.readFileSync('coverage-analysis.md', 'utf8');
	} catch (e) {
	baseAnalysis = '⚠️ Could not generate base coverage analysis';
	}

	try {
	comprehensiveAnalysis = fs.readFileSync('comprehensive-analysis.md', 'utf8');
	} catch (e) {
	comprehensiveAnalysis = '⚠️ Could not generate comprehensive analysis with SmartBear MCP tools';
	}

	try {
	prRecommendations = fs.readFileSync('pr-recommendations.md', 'utf8');
	} catch (e) {
	prRecommendations = '';
	}

	try {
	generatedTests = fs.readFileSync('generated-pact-tests.ts', 'utf8');
	} catch (e) {
	generatedTests = '';
	}

	// Get coverage statistics
	try {
	const coverageData = JSON.parse(fs.readFileSync('coverage-report.json', 'utf8'));
	const stats = coverageData.summary;
	coverageStats = `
	📊 Quick Stats:
	- 🎯 Coverage: ${stats.coveragePercentage}%
	- 📝 API Methods: ${stats.totalApiMethods}
	- ✅ Covered: ${stats.coveredMethods}
	- ❌ Missing: ${stats.totalApiMethods - stats.coveredMethods}
	- 🔄 Pact Interactions: ${stats.totalPactInteractions}
	`;
	} catch (e) {
	coverageStats = '⚠️ Could not load coverage statistics';
	}

	// Determine urgency level based on coverage
	const coveragePercent = "${{ steps.coverage-analysis.outputs.coverage_percent }}";
	let urgencyIcon = '🟢';
	let urgencyText = 'Good';

	if (coveragePercent < 50) {
	urgencyIcon = '🔴';
	urgencyText = 'Critical';
	} else if (coveragePercent < 80) {
	urgencyIcon = '🟡';
	urgencyText = 'Needs Attention';
	}

	// Create comprehensive comment
	const comment = `
	## 🤖 AI-Powered Pact Coverage Analysis ${urgencyIcon}

	Status: ${urgencyText} (${coveragePercent}% coverage)

	${coverageStats}

	<details>
	<summary>📋 <strong>Base Coverage Analysis</strong></summary>

	${baseAnalysis}

	</details>

	<details>
	<summary>🔬 <strong>SmartBear MCP Comprehensive Analysis</strong></summary>

	${comprehensiveAnalysis}

	</details>

	${prRecommendations ? `
	<details>
	<summary>🎯 <strong>PR-Specific Recommendations</strong></summary>

	${prRecommendations}

	</details>
	` : ''}

	${generatedTests ? `
	<details>
	<summary>💻 <strong>Generated Pact Test Code</strong></summary>

	The following Pact test code was generated using SmartBear MCP tools to improve coverage:

	\`\`\`typescript
	${generatedTests.length > 2000 ? generatedTests.substring(0, 2000) + '\n\n// ... (truncated, see full code in workflow artifacts)' : generatedTests}
	\`\`\`

	</details>
	` : ''}

	## � Next Steps

	${coveragePercent < 80 ? `
	### ⚠️ Action Required (Coverage < 80%)
	- [ ] Priority: Add missing Pact tests for uncovered API methods
	- [ ] Review: Check the SmartBear MCP generated test code above
	- [ ] Implement: Use the generated tests as a starting point
	` : ''}

	### 📋 General Tasks
	- [ ] Review coverage analysis and recommendations above
	- [ ] Add missing Pact test scenarios (especially error cases)
	- [ ] Ensure all API client methods have corresponding Pact tests
	- [ ] Verify error scenarios (400, 401, 404) are properly covered
	- [ ] Run \`npm run test:pact\` locally to validate any changes

	### 🔧 Quick Commands

	Generate comprehensive Pact tests using Claude + SmartBear MCP:
	\`\`\`bash
	claude -p --allowedTools "Read,Write,Bash,mcp__smartbear__generate_pact" --permission-mode acceptEdits "Create or update the pact tests in this project with the latest changes in the API client. Use SmartBear MCP tools to generate the updates, passing in the relevant files (api client, domain models, OpenAPI specification, templates, instructions) for context. Call the MCP tools once for each endpoint, and combine the results. Run the tests and fix any errors"
	\`\`\`

	Run Pact tests:
	\`\`\`bash
	npm run test:pact
	\`\`\`

	Analyze specific endpoint coverage:
	\`\`\`bash
	node .github/scripts/analyze-pact-coverage.js
	\`\`\`

	---

	<sub>🔄 This analysis was generated by Claude AI with SmartBear MCP contract testing tools \| Updated: ${new Date().toISOString()}</sub>

	<sub>💡 Tip: The SmartBear MCP tools provide AI-powered contract test generation and review capabilities specifically designed for API testing workflows.</sub>
	`;

	// Post comment
	await github.rest.issues.createComment({
	issue_number: context.issue.number,
	owner: context.repo.owner,
	repo: context.repo.repo,
	body: comment
	});

	// Add labels based on coverage
	const labels = [];
	if (coveragePercent < 50) {
	labels.push('pact-coverage-critical');
	} else if (coveragePercent < 80) {
	labels.push('pact-coverage-needs-attention');
	} else {
	labels.push('pact-coverage-good');
	}

	// Add general label
	labels.push('automated-pact-analysis');

	try {
	await github.rest.issues.addLabels({
	issue_number: context.issue.number,
	owner: context.repo.owner,
	repo: context.repo.repo,
	labels: labels
	});
	} catch (labelError) {
	console.log('Could not add labels (labels may not exist):', labelError.message);
	}

	- name: Upload Analysis Artifacts
	uses: actions/upload-artifact@v3
	if: always()
	with:
	name: pact-coverage-analysis-${{ github.event.pull_request.number }}
	path: \|
	coverage-report.json
	coverage-analysis.md
	comprehensive-analysis.md
	pr-recommendations.md
	generated-pact-tests.ts
	retention-days: 30

	# Only deploy from master
	deploy:
	runs-on: ubuntu-latest
	needs: can-i-deploy
	steps:
	- uses: actions/checkout@v3
	- run: docker pull pactfoundation/pact-cli:latest
	- name: Deploy
	run: GIT_BRANCH=${GIT_REF:11} make deploy
	if: github.ref == 'refs/heads/master' \|\| github.ref == 'refs/heads/test'

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

wip: test AI generated PR reviews #446

Workflow file

wip: test AI generated PR reviews #446

Uh oh!

Workflow file for this run