From 8b2461075d09c8b1432dbf4ec5b84331bd95f3de Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Wed, 16 Sep 2026 16:09:23 +0900 Subject: [PATCH] Add --locked flag to verify lockfile is in sync --- CHANGELOG.md | 6 + cmd/buf/internal/command/breaking/breaking.go | 4 + cmd/buf/internal/command/build/build.go | 4 + cmd/buf/internal/command/export/export.go | 4 + cmd/buf/internal/command/generate/generate.go | 7 + cmd/buf/internal/command/lint/lint.go | 4 + cmd/buf/internal/command/lsfiles/lsfiles.go | 4 + cmd/buf/internal/command/push/push.go | 4 + cmd/buf/internal/command/stats/stats.go | 4 + private/buf/bufcli/flags_args.go | 11 + private/buf/bufctl/controller.go | 49 ++- private/buf/bufctl/option.go | 11 + private/buf/bufworkspace/out_of_sync_dep.go | 340 ++++++++++++++++++ .../buf/bufworkspace/out_of_sync_dep_test.go | 249 +++++++++++++ .../buf/bufworkspace/workspace_dep_manager.go | 5 +- 15 files changed, 700 insertions(+), 6 deletions(-) create mode 100644 private/buf/bufworkspace/out_of_sync_dep.go create mode 100644 private/buf/bufworkspace/out_of_sync_dep_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 76c945721b..f7ec6a0fb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ - Add `--stdin-filepath` flag to `buf format`, which reads a single `.proto` file from stdin and writes the formatted result to stdout. The path is not read from disk, and is only used to report parse errors and diffs. +- Add `--locked` flag to `buf build`, `buf breaking`, `buf export`, `buf generate`, + `buf lint`, `buf ls-files`, `buf push`, and `buf stats`, which errors if the `buf.lock` + of the input does not satisfy its `buf.yaml`. A module dependency, remote plugin, or + remote policy declared with an explicit reference, such as + `buf.build/bufbuild/protovalidate:v0.14.1`, must be pinned to the commit that the + reference resolves to. ## [v1.73.0] - 2026-09-11 diff --git a/cmd/buf/internal/command/breaking/breaking.go b/cmd/buf/internal/command/breaking/breaking.go index b539b7429a..eef1a8ad43 100644 --- a/cmd/buf/internal/command/breaking/breaking.go +++ b/cmd/buf/internal/command/breaking/breaking.go @@ -53,6 +53,7 @@ const ( againstRegistryFlagName = "against-registry" excludePathsFlagName = "exclude-path" disableSymlinksFlagName = "disable-symlinks" + lockedFlagName = "locked" ) // NewCommand returns a new Command. @@ -92,6 +93,7 @@ type flags struct { AgainstRegistry bool ExcludePaths []string DisableSymlinks bool + Locked bool // special InputHashtag string } @@ -105,6 +107,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { bufcli.BindInputHashtag(flagSet, &f.InputHashtag) bufcli.BindExcludePaths(flagSet, &f.ExcludePaths, excludePathsFlagName) bufcli.BindDisableSymlinks(flagSet, &f.DisableSymlinks, disableSymlinksFlagName) + bufcli.BindLocked(flagSet, &f.Locked, lockedFlagName) flagSet.StringVar( &f.ErrorFormat, errorFormatFlagName, @@ -201,6 +204,7 @@ func run( wasmRuntime, bufctl.WithTargetPaths(flags.Paths, flags.ExcludePaths), bufctl.WithConfigOverride(flags.Config), + bufctl.WithLocked(flags.Locked), ) if err != nil { return err diff --git a/cmd/buf/internal/command/build/build.go b/cmd/buf/internal/command/build/build.go index f40b186a72..184afb77b3 100644 --- a/cmd/buf/internal/command/build/build.go +++ b/cmd/buf/internal/command/build/build.go @@ -43,6 +43,7 @@ const ( configFlagName = "config" excludePathsFlagName = "exclude-path" disableSymlinksFlagName = "disable-symlinks" + lockedFlagName = "locked" typeFlagName = "type" ) @@ -80,6 +81,7 @@ type flags struct { Config string ExcludePaths []string DisableSymlinks bool + Locked bool Types []string // special InputHashtag string @@ -97,6 +99,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { bufcli.BindPaths(flagSet, &f.Paths, pathsFlagName) bufcli.BindExcludePaths(flagSet, &f.ExcludePaths, excludePathsFlagName) bufcli.BindDisableSymlinks(flagSet, &f.DisableSymlinks, disableSymlinksFlagName) + bufcli.BindLocked(flagSet, &f.Locked, lockedFlagName) flagSet.BoolVar( &f.ExcludeSourceRetentionOptions, excludeSourceRetentionOptionsFlagName, @@ -164,6 +167,7 @@ func run( bufctl.WithImageExcludeImports(flags.ExcludeImports), bufctl.WithImageIncludeTypes(flags.Types), bufctl.WithConfigOverride(flags.Config), + bufctl.WithLocked(flags.Locked), ) if err != nil { return err diff --git a/cmd/buf/internal/command/export/export.go b/cmd/buf/internal/command/export/export.go index cb4c1df4a1..1ae5b685c3 100644 --- a/cmd/buf/internal/command/export/export.go +++ b/cmd/buf/internal/command/export/export.go @@ -44,6 +44,7 @@ const ( configFlagName = "config" excludePathsFlagName = "exclude-path" disableSymlinksFlagName = "disable-symlinks" + lockedFlagName = "locked" allFlagName = "all" ) @@ -103,6 +104,7 @@ type flags struct { Config string ExcludePaths []string DisableSymlinks bool + Locked bool All bool // special @@ -115,6 +117,7 @@ func newFlags() *flags { func (f *flags) Bind(flagSet *pflag.FlagSet) { bufcli.BindDisableSymlinks(flagSet, &f.DisableSymlinks, disableSymlinksFlagName) + bufcli.BindLocked(flagSet, &f.Locked, lockedFlagName) bufcli.BindInputHashtag(flagSet, &f.InputHashtag) bufcli.BindExcludeImports(flagSet, &f.ExcludeImports, excludeImportsFlagName) bufcli.BindPaths(flagSet, &f.Paths, pathsFlagName) @@ -162,6 +165,7 @@ func run( input, bufctl.WithTargetPaths(flags.Paths, flags.ExcludePaths), bufctl.WithConfigOverride(flags.Config), + bufctl.WithLocked(flags.Locked), ) if err != nil { return err diff --git a/cmd/buf/internal/command/generate/generate.go b/cmd/buf/internal/command/generate/generate.go index 706f4bde26..376c7f88d5 100644 --- a/cmd/buf/internal/command/generate/generate.go +++ b/cmd/buf/internal/command/generate/generate.go @@ -50,6 +50,7 @@ const ( includeWKTFlagName = "include-wkt" excludePathsFlagName = "exclude-path" disableSymlinksFlagName = "disable-symlinks" + lockedFlagName = "locked" typeFlagName = "type" typeDeprecatedFlagName = "include-types" excludeTypeFlagName = "exclude-type" @@ -416,6 +417,7 @@ type flags struct { IncludeWKTOverride *bool ExcludePaths []string DisableSymlinks bool + Locked bool // We may be able to bind two flags to one string slice but I don't // want to find out what will break if we do. Types []string @@ -431,6 +433,7 @@ func newFlags() *flags { func (f *flags) Bind(flagSet *pflag.FlagSet) { bufcli.BindDisableSymlinks(flagSet, &f.DisableSymlinks, disableSymlinksFlagName) + bufcli.BindLocked(flagSet, &f.Locked, lockedFlagName) bufcli.BindInputHashtag(flagSet, &f.InputHashtag) bufcli.BindPaths(flagSet, &f.Paths, pathsFlagName) bufcli.BindExcludePaths(flagSet, &f.ExcludePaths, excludePathsFlagName) @@ -557,6 +560,7 @@ func run( flags.ExcludePaths, append(flags.Types, flags.TypesDeprecated...), flags.ExcludeTypes, + flags.Locked, ) if err != nil { return err @@ -632,6 +636,7 @@ func getInputImages( excludePathsOverride []string, includeTypesOverride []string, excludeTypesOverride []string, + locked bool, ) ([]bufimage.Image, error) { // If input is specified on the command line, we use that. If input is not // specified on the command line, use the default input. @@ -658,6 +663,7 @@ func getInputImages( bufctl.WithTargetPaths(targetPathsOverride, excludePathsOverride), bufctl.WithImageIncludeTypes(includeTypes), bufctl.WithImageExcludeTypes(excludeTypes), + bufctl.WithLocked(locked), ) if err != nil { return nil, err @@ -689,6 +695,7 @@ func getInputImages( bufctl.WithTargetPaths(targetPaths, excludePaths), bufctl.WithImageIncludeTypes(includeTypes), bufctl.WithImageExcludeTypes(excludeTypes), + bufctl.WithLocked(locked), ) if err != nil { return nil, err diff --git a/cmd/buf/internal/command/lint/lint.go b/cmd/buf/internal/command/lint/lint.go index 732e5a1f90..b79ec34d7e 100644 --- a/cmd/buf/internal/command/lint/lint.go +++ b/cmd/buf/internal/command/lint/lint.go @@ -37,6 +37,7 @@ const ( pathsFlagName = "path" excludePathsFlagName = "exclude-path" disableSymlinksFlagName = "disable-symlinks" + lockedFlagName = "locked" ) // NewCommand returns a new Command. @@ -68,6 +69,7 @@ type flags struct { Paths []string ExcludePaths []string DisableSymlinks bool + Locked bool // special InputHashtag string } @@ -81,6 +83,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { bufcli.BindPaths(flagSet, &f.Paths, pathsFlagName) bufcli.BindExcludePaths(flagSet, &f.ExcludePaths, excludePathsFlagName) bufcli.BindDisableSymlinks(flagSet, &f.DisableSymlinks, disableSymlinksFlagName) + bufcli.BindLocked(flagSet, &f.Locked, lockedFlagName) flagSet.StringVar( &f.ErrorFormat, errorFormatFlagName, @@ -138,6 +141,7 @@ func run( wasmRuntime, bufctl.WithTargetPaths(flags.Paths, flags.ExcludePaths), bufctl.WithConfigOverride(flags.Config), + bufctl.WithLocked(flags.Locked), ) if err != nil { return err diff --git a/cmd/buf/internal/command/lsfiles/lsfiles.go b/cmd/buf/internal/command/lsfiles/lsfiles.go index 305a76bc80..42392ee71b 100644 --- a/cmd/buf/internal/command/lsfiles/lsfiles.go +++ b/cmd/buf/internal/command/lsfiles/lsfiles.go @@ -45,6 +45,7 @@ const ( pathsFlagName = "path" excludePathsFlagName = "exclude-path" disableSymlinksFlagName = "disable-symlinks" + lockedFlagName = "locked" asImportPathsFlagName = "as-import-paths" formatText = "text" @@ -94,6 +95,7 @@ type flags struct { Paths []string ExcludePaths []string DisableSymlinks bool + Locked bool // Deprecated. This flag no longer has any effect as we don't build images anymore. ErrorFormat string // Deprecated @@ -111,6 +113,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { bufcli.BindPaths(flagSet, &f.Paths, pathsFlagName) bufcli.BindExcludePaths(flagSet, &f.ExcludePaths, excludePathsFlagName) bufcli.BindDisableSymlinks(flagSet, &f.DisableSymlinks, disableSymlinksFlagName) + bufcli.BindLocked(flagSet, &f.Locked, lockedFlagName) flagSet.StringVar( &f.Config, configFlagName, @@ -187,6 +190,7 @@ func run( input, bufctl.WithTargetPaths(flags.Paths, flags.ExcludePaths), bufctl.WithConfigOverride(flags.Config), + bufctl.WithLocked(flags.Locked), ) if err != nil { return err diff --git a/cmd/buf/internal/command/push/push.go b/cmd/buf/internal/command/push/push.go index c89c2fd268..d55368b4c2 100644 --- a/cmd/buf/internal/command/push/push.go +++ b/cmd/buf/internal/command/push/push.go @@ -43,6 +43,7 @@ const ( labelFlagName = "label" errorFormatFlagName = "error-format" disableSymlinksFlagName = "disable-symlinks" + lockedFlagName = "locked" createFlagName = "create" createVisibilityFlagName = "create-visibility" createDefaultLabelFlagName = "create-default-label" @@ -97,6 +98,7 @@ type flags struct { Labels []string ErrorFormat string DisableSymlinks bool + Locked bool Create bool CreateVisibility string CreateDefaultLabel string @@ -114,6 +116,7 @@ func newFlags() *flags { func (f *flags) Bind(flagSet *pflag.FlagSet) { bufcli.BindInputHashtag(flagSet, &f.InputHashtag) bufcli.BindDisableSymlinks(flagSet, &f.DisableSymlinks, disableSymlinksFlagName) + bufcli.BindLocked(flagSet, &f.Locked, lockedFlagName) bufcli.BindCreateVisibility(flagSet, &f.CreateVisibility, createVisibilityFlagName, createFlagName) flagSet.StringSliceVar( &f.Labels, @@ -324,6 +327,7 @@ func getBuildableWorkspace( // that we don't want to deal with. If we have a v1 workspace, just outlaw pushing the whole // workspace, and force people into the pre-refactor behavior. bufctl.WithIgnoreAndDisallowV1BufWorkYAMLs(), + bufctl.WithLocked(flags.Locked), ) if err != nil { return nil, err diff --git a/cmd/buf/internal/command/stats/stats.go b/cmd/buf/internal/command/stats/stats.go index 3d2bcd1554..cc1b83de39 100644 --- a/cmd/buf/internal/command/stats/stats.go +++ b/cmd/buf/internal/command/stats/stats.go @@ -33,6 +33,7 @@ import ( const ( formatFlagName = "format" disableSymlinksFlagName = "disable-symlinks" + lockedFlagName = "locked" ) // NewCommand returns a new Command. @@ -61,6 +62,7 @@ func NewCommand( type flags struct { Format string DisableSymlinks bool + Locked bool // special InputHashtag string @@ -78,6 +80,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { fmt.Sprintf(`The output format to use. Must be one of %s`, bufprint.AllFormatsString), ) bufcli.BindDisableSymlinks(flagSet, &f.DisableSymlinks, disableSymlinksFlagName) + bufcli.BindLocked(flagSet, &f.Locked, lockedFlagName) bufcli.BindInputHashtag(flagSet, &f.InputHashtag) } @@ -104,6 +107,7 @@ func run( workspace, err := controller.GetWorkspace( ctx, input, + bufctl.WithLocked(flags.Locked), ) if err != nil { return err diff --git a/private/buf/bufcli/flags_args.go b/private/buf/bufcli/flags_args.go index 5d18766a98..653f0e10c8 100644 --- a/private/buf/bufcli/flags_args.go +++ b/private/buf/bufcli/flags_args.go @@ -148,6 +148,17 @@ By default, symlinks are followed in this CLI, but never followed on the Buf Sch ) } +// BindLocked binds the locked flag. +func BindLocked(flagSet *pflag.FlagSet, addr *bool, flagName string) { + flagSet.BoolVar( + addr, + flagName, + false, + `Error if the buf.lock is out of sync with the buf.yaml +A module dependency, remote plugin, or remote policy declared in the buf.yaml with an explicit reference must be pinned to the commit that the reference resolves to in the buf.lock. Run "buf dep update", "buf plugin update", or "buf policy update" to update the buf.lock`, + ) +} + // BindVisibility binds the visibility flag. func BindVisibility(flagSet *pflag.FlagSet, addr *string, flagName string, emptyDefault bool) { defaultVisibility := privateVisibility diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index d37e5498d4..9c3e7bbcf4 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -204,6 +204,7 @@ func NewController( type controller struct { logger *slog.Logger container app.EnvStdioContainer + moduleKeyProvider bufmodule.ModuleKeyProvider moduleDataProvider bufmodule.ModuleDataProvider graphProvider bufmodule.GraphProvider commitProvider bufmodule.CommitProvider @@ -247,6 +248,7 @@ func newController( logger: logger, container: container, graphProvider: graphProvider, + moduleKeyProvider: moduleKeyProvider, moduleDataProvider: moduleDataProvider, commitProvider: commitProvider, pluginKeyProvider: pluginKeyProvider, @@ -975,6 +977,30 @@ func (c *controller) getImageForWorkspace( return image, nil } +// checkLocked verifies that the buf.lock of the Workspace is in sync with its buf.yaml, +// if the locked FunctionOption was set. This covers the module, remote plugin, and remote +// policy deps declared in the buf.yaml. +func (c *controller) checkLocked( + ctx context.Context, + workspace bufworkspace.Workspace, + functionOptions *functionOptions, +) error { + if !functionOptions.locked { + return nil + } + outOfSyncDeps, err := bufworkspace.OutOfSyncDepsForWorkspace( + ctx, + c.moduleKeyProvider, + c.pluginKeyProvider, + c.policyKeyProvider, + workspace, + ) + if err != nil { + return err + } + return bufworkspace.NewOutOfSyncDepsError(outOfSyncDeps) +} + func (c *controller) getWorkspaceForProtoFileRef( ctx context.Context, protoFileRef buffetch.ProtoFileRef, @@ -1021,12 +1047,19 @@ func (c *controller) getWorkspaceForProtoFileRef( bufworkspace.WithIgnoreAndDisallowV1BufWorkYAMLs(), ) } - return c.workspaceProvider.GetWorkspaceForBucket( + workspace, err := c.workspaceProvider.GetWorkspaceForBucket( ctx, readBucketCloser, bucketTargeting, options..., ) + if err != nil { + return nil, err + } + if err := c.checkLocked(ctx, workspace, functionOptions); err != nil { + return nil, err + } + return workspace, nil } func (c *controller) getWorkspaceForSourceRef( @@ -1057,12 +1090,19 @@ func (c *controller) getWorkspaceForSourceRef( bufworkspace.WithIgnoreAndDisallowV1BufWorkYAMLs(), ) } - return c.workspaceProvider.GetWorkspaceForBucket( + workspace, err := c.workspaceProvider.GetWorkspaceForBucket( ctx, readBucketCloser, bucketTargeting, options..., ) + if err != nil { + return nil, err + } + if err := c.checkLocked(ctx, workspace, functionOptions); err != nil { + return nil, err + } + return workspace, nil } func (c *controller) getWorkspaceDepManagerForDirRef( @@ -1093,6 +1133,11 @@ func (c *controller) getWorkspaceForModuleRef( moduleRef buffetch.ModuleRef, functionOptions *functionOptions, ) (bufworkspace.Workspace, error) { + if functionOptions.locked { + // A module reference has no buf.yaml or buf.lock of its own to compare. + // TODO FUTURE: Feed flag names through to here. + return nil, errors.New("--locked is not valid for use with module references") + } moduleKey, err := c.buffetchReader.GetModuleKey(ctx, c.container, moduleRef) if err != nil { return nil, err diff --git a/private/buf/bufctl/option.go b/private/buf/bufctl/option.go index 74760444df..1f2adc6a49 100644 --- a/private/buf/bufctl/option.go +++ b/private/buf/bufctl/option.go @@ -132,6 +132,16 @@ func WithIgnoreAndDisallowV1BufWorkYAMLs() FunctionOption { } } +// WithLocked returns a new FunctionOption that says to verify that the buf.lock of the +// workspace is in sync with its buf.yaml, and to error if it is not. +// +// See bufworkspace.OutOfSyncDepsForWorkspace for what is and is not compared. +func WithLocked(locked bool) FunctionOption { + return func(functionOptions *functionOptions) { + functionOptions.locked = locked + } +} + // WithMessageValidation returns a new FunctionOption that says to validate the // message as it is being read. // @@ -157,6 +167,7 @@ type functionOptions struct { imageAsFileDescriptorSet bool configOverride string ignoreAndDisallowV1BufWorkYAMLs bool + locked bool messageValidation bool } diff --git a/private/buf/bufworkspace/out_of_sync_dep.go b/private/buf/bufworkspace/out_of_sync_dep.go new file mode 100644 index 0000000000..6d2ab0c7bd --- /dev/null +++ b/private/buf/bufworkspace/out_of_sync_dep.go @@ -0,0 +1,340 @@ +// Copyright 2020-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufworkspace + +import ( + "context" + "errors" + "fmt" + "slices" + "strconv" + "strings" + + "buf.build/go/standard/xslices" + "buf.build/go/standard/xstrings" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufpolicy" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/bufbuild/buf/private/pkg/uuidutil" + "github.com/google/uuid" +) + +const ( + // OutOfSyncDepTypeModule says that the dep is a module declared in the buf.yaml deps. + // + // These are updated with "buf dep update". + OutOfSyncDepTypeModule OutOfSyncDepType = iota + 1 + // OutOfSyncDepTypePlugin says that the dep is a remote plugin declared in the buf.yaml + // plugins. + // + // These are updated with "buf plugin update". + OutOfSyncDepTypePlugin + // OutOfSyncDepTypePolicy says that the dep is a remote policy declared in the buf.yaml + // policies. + // + // These are updated with "buf policy update". + OutOfSyncDepTypePolicy +) + +// OutOfSyncDepType is the type of dep that is out of sync. +type OutOfSyncDepType int + +// String returns the name of the dep type as it is referred to in user-facing messages. +func (o OutOfSyncDepType) String() string { + switch o { + case OutOfSyncDepTypeModule: + return "module" + case OutOfSyncDepTypePlugin: + return "plugin" + case OutOfSyncDepTypePolicy: + return "policy" + default: + return strconv.Itoa(int(o)) + } +} + +// OutOfSyncDep is a dep declared in a buf.yaml that the buf.lock does not satisfy. +// +// A dep declared with an explicit reference, such as +// "buf.build/bufbuild/protovalidate:v0.14.1", constrains the buf.lock to the commit that +// the reference resolves to. If the buf.lock pins a different commit, the buf.lock is out +// of sync. +type OutOfSyncDep interface { + // Ref is the dep as declared in the buf.yaml. + // + // Always present. + Ref() bufparse.Ref + // Type is the type of dep that is out of sync. + // + // Always present. + Type() OutOfSyncDepType + // ExpectedCommitID is the commit that Ref resolves to. + // + // Always present. + ExpectedCommitID() uuid.UUID + // ExistingCommitID is the commit that the buf.lock pins the dep to. + // + // Always present. + ExistingCommitID() uuid.UUID + + isOutOfSyncDep() +} + +// OutOfSyncDepsForWorkspace returns the OutOfSyncDeps for the Workspace, covering the +// module, remote plugin, and remote policy deps declared in the buf.yaml. +// +// Only deps that are declared in the buf.yaml with an explicit reference and that are +// present in the buf.lock are compared. +// +// The remote plugins that a policy itself declares are not compared. For a remote policy +// these follow from the policy commit, which is compared. For a local policy these are +// declared in a local buf.policy.yaml, which is not part of the buf.yaml. +// +// This makes a call to the BSR to resolve each declared reference to a commit. +func OutOfSyncDepsForWorkspace( + ctx context.Context, + moduleKeyProvider bufmodule.ModuleKeyProvider, + pluginKeyProvider bufplugin.PluginKeyProvider, + policyKeyProvider bufpolicy.PolicyKeyProvider, + workspace Workspace, +) ([]OutOfSyncDep, error) { + moduleOutOfSyncDeps, err := outOfSyncDeps( + ctx, + OutOfSyncDepTypeModule, + workspace.ConfiguredDepModuleRefs(), + bufmodule.ModuleSetRemoteModules(workspace), + moduleKeysForRefsFunc(moduleKeyProvider, bufLockFileDigestTypeForIsV2(workspace.IsV2())), + ) + if err != nil { + return nil, err + } + pluginOutOfSyncDeps, err := outOfSyncDeps( + ctx, + OutOfSyncDepTypePlugin, + refsForConfigs(workspace.PluginConfigs(), bufconfig.PluginConfig.Ref), + workspace.RemotePluginKeys(), + pluginKeysForRefsFunc(pluginKeyProvider), + ) + if err != nil { + return nil, err + } + policyOutOfSyncDeps, err := outOfSyncDeps( + ctx, + OutOfSyncDepTypePolicy, + refsForConfigs(workspace.PolicyConfigs(), bufconfig.PolicyConfig.Ref), + workspace.RemotePolicyKeys(), + policyKeysForRefsFunc(policyKeyProvider), + ) + if err != nil { + return nil, err + } + return slices.Concat(moduleOutOfSyncDeps, pluginOutOfSyncDeps, policyOutOfSyncDeps), nil +} + +// NewOutOfSyncDepsError returns an error describing the given OutOfSyncDeps. +// +// Returns nil if outOfSyncDeps is empty. +func NewOutOfSyncDepsError(outOfSyncDeps []OutOfSyncDep) error { + if len(outOfSyncDeps) == 0 { + return nil + } + var builder strings.Builder + _, _ = builder.WriteString("buf.lock is out of sync with buf.yaml:") + for _, outOfSyncDep := range outOfSyncDeps { + _, _ = fmt.Fprintf( + &builder, + "\n\t%s %s is declared in buf.yaml, but buf.lock pins commit %s instead of %s", + outOfSyncDep.Type().String(), + outOfSyncDep.Ref().String(), + uuidutil.ToDashless(outOfSyncDep.ExistingCommitID()), + uuidutil.ToDashless(outOfSyncDep.ExpectedCommitID()), + ) + } + updateCommands := xslices.ToUniqueSorted( + xslices.Map( + outOfSyncDeps, + func(outOfSyncDep OutOfSyncDep) string { + return updateCommandForOutOfSyncDepType(outOfSyncDep.Type()) + }, + ), + ) + _, _ = fmt.Fprintf( + &builder, + "\nRun %s to update buf.lock.", + xstrings.SliceToHumanStringQuoted(updateCommands), + ) + return errors.New(builder.String()) +} + +// *** PRIVATE *** + +// outOfSyncDepKey is the shape shared by bufmodule.Module, bufmodule.ModuleKey, +// bufplugin.PluginKey, and bufpolicy.PolicyKey. +type outOfSyncDepKey interface { + FullName() bufparse.FullName + CommitID() uuid.UUID +} + +func outOfSyncDeps[ExistingKey outOfSyncDepKey, ResolvedKey outOfSyncDepKey]( + ctx context.Context, + outOfSyncDepType OutOfSyncDepType, + configuredRefs []bufparse.Ref, + existingKeys []ExistingKey, + getKeysForRefs func(context.Context, []bufparse.Ref) ([]ResolvedKey, error), +) ([]OutOfSyncDep, error) { + fullNameStringToExistingCommitID := make(map[string]uuid.UUID, len(existingKeys)) + for _, existingKey := range existingKeys { + fullName := existingKey.FullName() + if fullName == nil { + continue + } + if commitID := existingKey.CommitID(); commitID != uuid.Nil { + fullNameStringToExistingCommitID[fullName.String()] = commitID + } + } + constrainedRefs := xslices.Filter( + configuredRefs, + func(ref bufparse.Ref) bool { + if ref.Ref() == "" { + return false + } + _, ok := fullNameStringToExistingCommitID[ref.FullName().String()] + return ok + }, + ) + if len(constrainedRefs) == 0 { + return nil, nil + } + resolvedKeys, err := getKeysForRefs(ctx, constrainedRefs) + if err != nil { + return nil, err + } + if len(resolvedKeys) != len(constrainedRefs) { + return nil, syserror.Newf( + "got %d keys for %d %s refs", + len(resolvedKeys), + len(constrainedRefs), + outOfSyncDepType.String(), + ) + } + var resultOutOfSyncDeps []OutOfSyncDep + for i, resolvedKey := range resolvedKeys { + ref := constrainedRefs[i] + existingCommitID := fullNameStringToExistingCommitID[ref.FullName().String()] + if resolvedKey.CommitID() == existingCommitID { + continue + } + resultOutOfSyncDeps = append( + resultOutOfSyncDeps, + newOutOfSyncDep(ref, outOfSyncDepType, resolvedKey.CommitID(), existingCommitID), + ) + } + return resultOutOfSyncDeps, nil +} + +func moduleKeysForRefsFunc( + moduleKeyProvider bufmodule.ModuleKeyProvider, + digestType bufmodule.DigestType, +) func(context.Context, []bufparse.Ref) ([]bufmodule.ModuleKey, error) { + return func(ctx context.Context, refs []bufparse.Ref) ([]bufmodule.ModuleKey, error) { + return moduleKeyProvider.GetModuleKeysForModuleRefs(ctx, refs, digestType) + } +} + +func pluginKeysForRefsFunc( + pluginKeyProvider bufplugin.PluginKeyProvider, +) func(context.Context, []bufparse.Ref) ([]bufplugin.PluginKey, error) { + return func(ctx context.Context, refs []bufparse.Ref) ([]bufplugin.PluginKey, error) { + return pluginKeyProvider.GetPluginKeysForPluginRefs(ctx, refs, bufplugin.DigestTypeP1) + } +} + +func policyKeysForRefsFunc( + policyKeyProvider bufpolicy.PolicyKeyProvider, +) func(context.Context, []bufparse.Ref) ([]bufpolicy.PolicyKey, error) { + return func(ctx context.Context, refs []bufparse.Ref) ([]bufpolicy.PolicyKey, error) { + return policyKeyProvider.GetPolicyKeysForPolicyRefs(ctx, refs, bufpolicy.DigestTypeO1) + } +} + +// refsForConfigs returns the Refs of the configs that are remote. +func refsForConfigs[Config any](configs []Config, getRef func(Config) bufparse.Ref) []bufparse.Ref { + return xslices.Filter( + xslices.Map(configs, getRef), + func(ref bufparse.Ref) bool { + return ref != nil + }, + ) +} + +func updateCommandForOutOfSyncDepType(outOfSyncDepType OutOfSyncDepType) string { + switch outOfSyncDepType { + case OutOfSyncDepTypePlugin: + return "buf plugin update" + case OutOfSyncDepTypePolicy: + return "buf policy update" + default: + return "buf dep update" + } +} + +func bufLockFileDigestTypeForIsV2(isV2 bool) bufmodule.DigestType { + if isV2 { + return bufmodule.DigestTypeB5 + } + return bufmodule.DigestTypeB4 +} + +type outOfSyncDep struct { + ref bufparse.Ref + outOfSyncDepType OutOfSyncDepType + expectedCommitID uuid.UUID + existingCommitID uuid.UUID +} + +func newOutOfSyncDep( + ref bufparse.Ref, + outOfSyncDepType OutOfSyncDepType, + expectedCommitID uuid.UUID, + existingCommitID uuid.UUID, +) *outOfSyncDep { + return &outOfSyncDep{ + ref: ref, + outOfSyncDepType: outOfSyncDepType, + expectedCommitID: expectedCommitID, + existingCommitID: existingCommitID, + } +} + +func (o *outOfSyncDep) Ref() bufparse.Ref { + return o.ref +} + +func (o *outOfSyncDep) Type() OutOfSyncDepType { + return o.outOfSyncDepType +} + +func (o *outOfSyncDep) ExpectedCommitID() uuid.UUID { + return o.expectedCommitID +} + +func (o *outOfSyncDep) ExistingCommitID() uuid.UUID { + return o.existingCommitID +} + +func (*outOfSyncDep) isOutOfSyncDep() {} diff --git a/private/buf/bufworkspace/out_of_sync_dep_test.go b/private/buf/bufworkspace/out_of_sync_dep_test.go new file mode 100644 index 0000000000..ada8a6e546 --- /dev/null +++ b/private/buf/bufworkspace/out_of_sync_dep_test.go @@ -0,0 +1,249 @@ +// Copyright 2020-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufworkspace + +import ( + "context" + "errors" + "io/fs" + "testing" + + "buf.build/go/standard/xslices" + "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + commitIDOne = uuid.MustParse("00000000-0000-0000-0000-000000000001") + commitIDTwo = uuid.MustParse("00000000-0000-0000-0000-000000000002") + commitIDThree = uuid.MustParse("00000000-0000-0000-0000-000000000003") +) + +func TestOutOfSyncDeps(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + // The refs as declared in the buf.yaml. + configuredRefStrings []string + // The commit that each ref resolves to on the BSR. + refStringToResolvedCommitID map[string]uuid.UUID + // The commit that each dep is pinned to in the buf.lock. + fullNameStringToExistingCommitID map[string]uuid.UUID + // The refs expected to be reported as out of sync. + expectedOutOfSyncRefStrings []string + }{ + { + name: "in sync", + configuredRefStrings: []string{"buf.build/foo/one:v1.0.0"}, + refStringToResolvedCommitID: map[string]uuid.UUID{ + "buf.build/foo/one:v1.0.0": commitIDOne, + }, + fullNameStringToExistingCommitID: map[string]uuid.UUID{ + "buf.build/foo/one": commitIDOne, + }, + }, + { + name: "pinned ref does not match buf.lock", + configuredRefStrings: []string{"buf.build/foo/one:v1.0.0"}, + refStringToResolvedCommitID: map[string]uuid.UUID{ + "buf.build/foo/one:v1.0.0": commitIDOne, + }, + fullNameStringToExistingCommitID: map[string]uuid.UUID{ + "buf.build/foo/one": commitIDTwo, + }, + expectedOutOfSyncRefStrings: []string{"buf.build/foo/one:v1.0.0"}, + }, + { + name: "unpinned ref is never out of sync", + configuredRefStrings: []string{"buf.build/foo/one"}, + refStringToResolvedCommitID: map[string]uuid.UUID{ + "buf.build/foo/one": commitIDOne, + }, + fullNameStringToExistingCommitID: map[string]uuid.UUID{ + "buf.build/foo/one": commitIDTwo, + }, + }, + { + name: "pinned ref pruned from buf.lock is not out of sync", + configuredRefStrings: []string{"buf.build/foo/one:v1.0.0"}, + refStringToResolvedCommitID: map[string]uuid.UUID{ + "buf.build/foo/one:v1.0.0": commitIDOne, + }, + fullNameStringToExistingCommitID: map[string]uuid.UUID{}, + }, + { + name: "only the out of sync deps are reported", + configuredRefStrings: []string{ + "buf.build/foo/one:v1.0.0", + "buf.build/foo/three:v3.0.0", + "buf.build/foo/two", + }, + refStringToResolvedCommitID: map[string]uuid.UUID{ + "buf.build/foo/one:v1.0.0": commitIDOne, + "buf.build/foo/two": commitIDTwo, + "buf.build/foo/three:v3.0.0": commitIDThree, + }, + fullNameStringToExistingCommitID: map[string]uuid.UUID{ + "buf.build/foo/one": commitIDOne, + "buf.build/foo/two": commitIDOne, + "buf.build/foo/three": commitIDOne, + }, + expectedOutOfSyncRefStrings: []string{"buf.build/foo/three:v3.0.0"}, + }, + { + name: "no deps", + fullNameStringToExistingCommitID: map[string]uuid.UUID{}, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + configuredRefs, err := xslices.MapError(testCase.configuredRefStrings, bufparse.ParseRef) + require.NoError(t, err) + existingKeys, err := testModuleKeysForFullNameStringToCommitID(testCase.fullNameStringToExistingCommitID) + require.NoError(t, err) + actualOutOfSyncDeps, err := outOfSyncDeps( + t.Context(), + OutOfSyncDepTypeModule, + configuredRefs, + existingKeys, + newTestKeysForRefsFunc(testCase.refStringToResolvedCommitID, nil), + ) + require.NoError(t, err) + actualOutOfSyncRefStrings := xslices.Map( + actualOutOfSyncDeps, + func(outOfSyncDep OutOfSyncDep) string { + return outOfSyncDep.Ref().String() + }, + ) + if len(testCase.expectedOutOfSyncRefStrings) == 0 { + assert.Empty(t, actualOutOfSyncRefStrings) + } else { + assert.Equal(t, testCase.expectedOutOfSyncRefStrings, actualOutOfSyncRefStrings) + } + }) + } +} + +func TestOutOfSyncDepsOnlyResolvesConstrainedDeps(t *testing.T) { + t.Parallel() + + // Resolving a ref is a call to the BSR - we should not be making one for a dep that + // cannot be out of sync. + var requestedRefStrings []string + configuredRefs, err := xslices.MapError( + []string{"buf.build/foo/one:v1.0.0", "buf.build/foo/two", "buf.build/foo/three:v3.0.0"}, + bufparse.ParseRef, + ) + require.NoError(t, err) + existingKeys, err := testModuleKeysForFullNameStringToCommitID( + map[string]uuid.UUID{ + "buf.build/foo/one": commitIDOne, + "buf.build/foo/two": commitIDTwo, + }, + ) + require.NoError(t, err) + _, err = outOfSyncDeps( + t.Context(), + OutOfSyncDepTypeModule, + configuredRefs, + existingKeys, + newTestKeysForRefsFunc( + map[string]uuid.UUID{"buf.build/foo/one:v1.0.0": commitIDOne}, + &requestedRefStrings, + ), + ) + require.NoError(t, err) + assert.Equal(t, []string{"buf.build/foo/one:v1.0.0"}, requestedRefStrings) +} + +func TestNewOutOfSyncDepsError(t *testing.T) { + t.Parallel() + + assert.NoError(t, NewOutOfSyncDepsError(nil)) + moduleRef, err := bufparse.ParseRef("buf.build/foo/one:v1.0.0") + require.NoError(t, err) + pluginRef, err := bufparse.ParseRef("buf.build/foo/two:v2.0.0") + require.NoError(t, err) + err = NewOutOfSyncDepsError( + []OutOfSyncDep{ + newOutOfSyncDep(moduleRef, OutOfSyncDepTypeModule, commitIDOne, commitIDTwo), + newOutOfSyncDep(pluginRef, OutOfSyncDepTypePlugin, commitIDTwo, commitIDThree), + }, + ) + require.Error(t, err) + assert.Equal( + t, + `buf.lock is out of sync with buf.yaml: + module buf.build/foo/one:v1.0.0 is declared in buf.yaml, but buf.lock pins commit 00000000000000000000000000000002 instead of 00000000000000000000000000000001 + plugin buf.build/foo/two:v2.0.0 is declared in buf.yaml, but buf.lock pins commit 00000000000000000000000000000003 instead of 00000000000000000000000000000002 +Run "buf dep update" and "buf plugin update" to update buf.lock.`, + err.Error(), + ) +} + +func testModuleKeysForFullNameStringToCommitID( + fullNameStringToCommitID map[string]uuid.UUID, +) ([]bufmodule.ModuleKey, error) { + moduleKeys := make([]bufmodule.ModuleKey, 0, len(fullNameStringToCommitID)) + for fullNameString, commitID := range fullNameStringToCommitID { + fullName, err := bufparse.ParseFullName(fullNameString) + if err != nil { + return nil, err + } + moduleKey, err := bufmodule.NewModuleKey(fullName, commitID, testDigestFunc) + if err != nil { + return nil, err + } + moduleKeys = append(moduleKeys, moduleKey) + } + return moduleKeys, nil +} + +// newTestKeysForRefsFunc resolves Refs to commits based on the full Ref string, so that a +// pinned Ref and an unpinned Ref for the same module can resolve to different commits. +// +// If requestedRefStrings is non-nil, each requested Ref string is appended to it. +func newTestKeysForRefsFunc( + refStringToCommitID map[string]uuid.UUID, + requestedRefStrings *[]string, +) func(context.Context, []bufparse.Ref) ([]bufmodule.ModuleKey, error) { + return func(_ context.Context, refs []bufparse.Ref) ([]bufmodule.ModuleKey, error) { + moduleKeys := make([]bufmodule.ModuleKey, len(refs)) + for i, ref := range refs { + if requestedRefStrings != nil { + *requestedRefStrings = append(*requestedRefStrings, ref.String()) + } + commitID, ok := refStringToCommitID[ref.String()] + if !ok { + return nil, &fs.PathError{Op: "read", Path: ref.String(), Err: fs.ErrNotExist} + } + moduleKey, err := bufmodule.NewModuleKey(ref.FullName(), commitID, testDigestFunc) + if err != nil { + return nil, err + } + moduleKeys[i] = moduleKey + } + return moduleKeys, nil + } +} + +func testDigestFunc() (bufmodule.Digest, error) { + return nil, errors.New("digest not available in test") +} diff --git a/private/buf/bufworkspace/workspace_dep_manager.go b/private/buf/bufworkspace/workspace_dep_manager.go index eb501cd91e..dd6f20a12e 100644 --- a/private/buf/bufworkspace/workspace_dep_manager.go +++ b/private/buf/bufworkspace/workspace_dep_manager.go @@ -310,10 +310,7 @@ func (w *workspaceDepManager) ConfiguredLocalPolicyNameToRemotePluginRefs(ctx co } func (w *workspaceDepManager) BufLockFileDigestType() bufmodule.DigestType { - if w.isV2 { - return bufmodule.DigestTypeB5 - } - return bufmodule.DigestTypeB4 + return bufLockFileDigestTypeForIsV2(w.isV2) } func (w *workspaceDepManager) ExistingBufLockFileDepModuleKeys(ctx context.Context) ([]bufmodule.ModuleKey, error) {