diff --git a/crates/ocync-sync/src/filter.rs b/crates/ocync-sync/src/filter.rs index ed1a63f..78113be 100644 --- a/crates/ocync-sync/src/filter.rs +++ b/crates/ocync-sync/src/filter.rs @@ -53,21 +53,39 @@ fn system_exclude_set() -> &'static GlobSet { /// /// All stages are AND (narrowing). Each stage reduces the set: /// `glob → semver → exclude → sort → latest → min_tags`. -/// Tags matching `include:` bypass the glob/semver pipeline (but are still -/// subject to user `exclude:`). +/// Tags matching `include:` bypass the glob/semver pipeline AND the soft +/// exclude tier (built-in + defaults). They are still subject to mapping +/// `exclude:` (the hard tier). +/// +/// # Exclude tiers +/// +/// Exclusion has two tiers: +/// +/// - **Soft tier** (built-in `SYSTEM_EXCLUDE` + caller-provided +/// [`defaults_exclude`](Self::defaults_exclude)): bypassable by +/// `include:`. Use for project-wide opinions like "drop `*-dev` unless +/// I say otherwise on a specific mapping." +/// - **Hard tier** ([`exclude`](Self::exclude)): blocks `include:` on the +/// same config. Use for absolute per-mapping denies. #[derive(Debug, Default)] pub struct FilterConfig { /// Always-include glob patterns. Tags matching any pattern survive - /// `glob:`/`semver:` filters and the system-exclude defaults. Not - /// subject to `sort:` or `latest:` truncation (those only cap the - /// `glob:`/`semver:` pipeline side). Subject to user `exclude:`. Same - /// syntax as `exclude:`. + /// `glob:`/`semver:` filters and the soft exclude tier (system + defaults). + /// Not subject to `sort:` or `latest:` truncation (those only cap the + /// `glob:`/`semver:` pipeline side). Subject to mapping + /// [`exclude`](Self::exclude). Same glob syntax as `exclude:`. pub include: Vec, /// Glob patterns (OR semantics). An empty list passes all tags through. pub glob: Vec, /// Semver version range constraint (e.g. `>=1.18.0`). pub semver: Option, - /// Exclude patterns (OR deny). + /// Soft-tier exclude patterns inherited from a `defaults:` block. + /// Bypassed by [`include`](Self::include), unlike + /// [`exclude`](Self::exclude). Behaves the same as the built-in + /// `SYSTEM_EXCLUDE` list. + pub defaults_exclude: Vec, + /// Hard-tier exclude patterns (OR deny). Blocks + /// [`include`](Self::include) on the same config. pub exclude: Vec, /// Sort order. pub sort: Option, @@ -125,8 +143,13 @@ impl FilterConfig { if let Some(ref s) = self.semver { parts.push(format!("semver {s}")); } - if !self.exclude.is_empty() { - parts.push(format!("exclude {}", self.exclude.join(","))); + if !self.exclude.is_empty() || !self.defaults_exclude.is_empty() { + // Defaults- and mapping-tier patterns share one summary clause. + // Dry-run carries the tier attribution; the INFO line stays tight. + let mut combined: Vec<&str> = + self.defaults_exclude.iter().map(String::as_str).collect(); + combined.extend(self.exclude.iter().map(String::as_str)); + parts.push(format!("exclude {}", combined.join(","))); } if !self.include.is_empty() { parts.push(format!("include {}", self.include.join(","))); @@ -170,6 +193,11 @@ impl FilterConfig { } else { Some(build_glob_set(&self.exclude)?) }; + let defaults_exclude_set = if self.defaults_exclude.is_empty() { + None + } else { + Some(build_glob_set(&self.defaults_exclude)?) + }; let sys_exclude = system_exclude_set(); let include_kept_refs: Vec<&str> = if self.include.is_empty() { @@ -243,46 +271,69 @@ impl FilterConfig { } } + // Exclude stage: three tiers, evaluated in order so the first match + // attributes the drop. Order doesn't change kept tags (they're all + // OR-deny); it only decides which DropKind a tag is reported under. + // Mapping (hard) is checked first because it represents the most + // specific user intent. let before_exclude = pipeline.len(); - let mut user_dropped: Vec = Vec::new(); - let mut sys_dropped: Vec = Vec::new(); + let mut mapping_dropped: Vec = Vec::new(); + let mut defaults_dropped: Vec = Vec::new(); + let mut builtin_dropped: Vec = Vec::new(); pipeline.retain(|t| { if let Some(ref s) = user_exclude_set { if s.is_match(t) { if track { - user_dropped.push((*t).to_owned()); + mapping_dropped.push((*t).to_owned()); + } + return false; + } + } + if let Some(ref s) = defaults_exclude_set { + if s.is_match(t) { + if track { + defaults_dropped.push((*t).to_owned()); } return false; } } if sys_exclude.is_match(t) { if track { - sys_dropped.push((*t).to_owned()); + builtin_dropped.push((*t).to_owned()); } return false; } true }); if track { - if !user_dropped.is_empty() { + if !mapping_dropped.is_empty() { drop_reasons.push(DropReason { - kind: DropKind::UserExclude { + kind: DropKind::MappingExclude { patterns: self.exclude.clone(), }, - count: user_dropped.len(), - samples: user_dropped, + count: mapping_dropped.len(), + samples: mapping_dropped, }); } - if !sys_dropped.is_empty() { + if !defaults_dropped.is_empty() { drop_reasons.push(DropReason { - kind: DropKind::SystemExclude, - count: sys_dropped.len(), - samples: sys_dropped, + kind: DropKind::DefaultsExclude { + patterns: self.defaults_exclude.clone(), + }, + count: defaults_dropped.len(), + samples: defaults_dropped, + }); + } + if !builtin_dropped.is_empty() { + drop_reasons.push(DropReason { + kind: DropKind::BuiltinExclude, + count: builtin_dropped.len(), + samples: builtin_dropped, }); } if before_exclude != pipeline.len() { pipeline_stages.push(StageDelta { - label: "exclude (user + system)".to_string(), + label: "exclude (mapping + defaults + built-in)".to_string(), count_in: before_exclude, count_out: pipeline.len(), }); @@ -431,13 +482,21 @@ pub enum DropKind { /// The configured version range string, e.g. `">=1.18.0"`. range: String, }, - /// Tag matched a user-configured `exclude:` pattern. - UserExclude { - /// User-configured exclude patterns (one or more). + /// Tag matched a per-mapping `exclude:` pattern (hard tier; blocks + /// `include:` on the same mapping). + MappingExclude { + /// Mapping-level exclude patterns (one or more). + patterns: Vec, + }, + /// Tag matched a `defaults.tags.exclude:` pattern (soft tier; bypassable + /// by `include:`). + DefaultsExclude { + /// Defaults-level exclude patterns (one or more). patterns: Vec, }, - /// Tag matched the built-in prerelease exclude list. - SystemExclude, + /// Tag matched the built-in prerelease exclude list (soft tier; + /// bypassable by `include:`). + BuiltinExclude, /// Tag fell off the end of the `latest: N` truncation. LatestCap { /// The configured `latest: N` value. @@ -450,10 +509,13 @@ impl fmt::Display for DropKind { match self { Self::Glob { patterns } => write!(f, "glob {}", patterns_label(patterns)), Self::Semver { range } => write!(f, "semver \"{range}\""), - Self::UserExclude { patterns } => { - write!(f, "user-exclude {}", patterns_label(patterns)) + Self::MappingExclude { patterns } => { + write!(f, "exclude (mapping) {}", patterns_label(patterns)) + } + Self::DefaultsExclude { patterns } => { + write!(f, "exclude (defaults) {}", patterns_label(patterns)) } - Self::SystemExclude => f.write_str("system-exclude"), + Self::BuiltinExclude => f.write_str("exclude (built-in)"), Self::LatestCap { limit } => write!(f, "over latest={limit} limit"), } } @@ -1178,6 +1240,108 @@ mod tests { assert!(!result.contains(&"latest".to_string())); } + // - defaults_exclude tier -------------------------------------------- + + /// `defaults_exclude` drops tags from the pipeline just like the + /// built-in system exclude. The mapping has no `exclude:` of its own, + /// so this proves defaults flow through. + #[test] + fn defaults_exclude_drops_tags() { + let tags = vec!["1.0.0", "1.0.0-dev", "1.0.0-r0"]; + let config = FilterConfig { + defaults_exclude: vec!["*-dev".into(), "*-r[0-9]*".into()], + ..FilterConfig::default() + }; + let result = config.apply(&tags).unwrap(); + assert_eq!(result, vec!["1.0.0".to_string()]); + } + + /// `include:` rescues a tag that `defaults_exclude` would drop. This is + /// the user-facing escape hatch for the project-wide exclude floor. + #[test] + fn include_overrides_defaults_exclude() { + let tags = vec!["latest", "latest-dev", "1.0.0", "1.0.0-dev"]; + let config = FilterConfig { + include: vec!["latest-dev".into()], + defaults_exclude: vec!["*-dev".into()], + ..FilterConfig::default() + }; + let result = config.apply(&tags).unwrap(); + assert!(result.contains(&"latest".to_string())); + assert!( + result.contains(&"latest-dev".to_string()), + "include should rescue latest-dev from defaults_exclude" + ); + assert!(result.contains(&"1.0.0".to_string())); + // 1.0.0-dev does not match include and is dropped by defaults_exclude. + assert!(!result.contains(&"1.0.0-dev".to_string())); + } + + /// Mapping-level `exclude:` is the hard tier: it blocks `include:` even + /// when a `defaults_exclude` is also configured. Negative assertion + /// preserving existing semantics. + #[test] + fn mapping_exclude_blocks_include_with_defaults_set() { + let tags = vec!["latest", "latest-dev"]; + let config = FilterConfig { + include: vec!["latest".into(), "latest-dev".into()], + defaults_exclude: vec!["*-dev".into()], + exclude: vec!["latest".into()], + ..FilterConfig::default() + }; + let result = config.apply(&tags).unwrap(); + // mapping.exclude = ["latest"] blocks include of "latest" + assert!(!result.contains(&"latest".to_string())); + // include still rescues latest-dev (matches defaults_exclude soft tier only) + assert!(result.contains(&"latest-dev".to_string())); + } + + /// `defaults_exclude` and `exclude` (mapping) both apply: their union + /// drops tags. Stacking, not replacement. + #[test] + fn defaults_and_mapping_exclude_stack() { + let tags = vec!["1.0.0", "1.0.0-dev", "1.0.0-slim"]; + let config = FilterConfig { + defaults_exclude: vec!["*-dev".into()], + exclude: vec!["*-slim".into()], + ..FilterConfig::default() + }; + let result = config.apply(&tags).unwrap(); + assert_eq!(result, vec!["1.0.0".to_string()]); + } + + /// Dry-run attribution: defaults-tier drops surface as + /// `DropKind::DefaultsExclude`, distinct from `MappingExclude` and + /// `BuiltinExclude`. The formatter relies on the variant to render + /// `(defaults)` / `(mapping)` / `(built-in)`. + #[test] + fn report_attributes_defaults_exclude_separately() { + let tags = vec!["1.0.0", "1.0.0-dev", "1.0.0-rc1", "1.0.0-slim"]; + let config = FilterConfig { + defaults_exclude: vec!["*-dev".into()], + exclude: vec!["*-slim".into()], + ..FilterConfig::default() + }; + let filtered = config.apply_with_report(&tags).unwrap(); + let kinds: Vec<&DropKind> = filtered.report.dropped.iter().map(|d| &d.kind).collect(); + assert!( + kinds + .iter() + .any(|k| matches!(k, DropKind::DefaultsExclude { .. })), + "missing DefaultsExclude in {kinds:?}" + ); + assert!( + kinds + .iter() + .any(|k| matches!(k, DropKind::MappingExclude { .. })), + "missing MappingExclude in {kinds:?}" + ); + assert!( + kinds.iter().any(|k| matches!(k, DropKind::BuiltinExclude)), + "missing BuiltinExclude (1.0.0-rc1 should hit it) in {kinds:?}" + ); + } + #[test] fn latest_n_does_not_cap_include() { // Pipeline has 5 candidates; latest:2 should keep only the top 2 of @@ -1398,12 +1562,12 @@ mod tests { assert!( kinds .iter() - .any(|k| matches!(k, DropKind::UserExclude { .. })), - "missing UserExclude in {kinds:?}" + .any(|k| matches!(k, DropKind::MappingExclude { .. })), + "missing MappingExclude in {kinds:?}" ); assert!( - kinds.iter().any(|k| matches!(k, DropKind::SystemExclude)), - "missing SystemExclude in {kinds:?}" + kinds.iter().any(|k| matches!(k, DropKind::BuiltinExclude)), + "missing BuiltinExclude in {kinds:?}" ); } @@ -1494,6 +1658,7 @@ mod tests { include: vec!["latest".into()], glob: vec!["v*".into()], semver: Some("^1".into()), + defaults_exclude: vec!["*-dev".into()], exclude: vec!["nightly".into()], sort: Some(SortOrder::Alpha), latest: Some(3), @@ -1504,7 +1669,7 @@ mod tests { "include latest", "glob v*", "semver ^1", - "exclude nightly", + "exclude *-dev,nightly", "sort alpha", "latest=3", "min_tags=2", diff --git a/docs/src/content/configuration.md b/docs/src/content/configuration.md index b65ee5b..a66521e 100644 --- a/docs/src/content/configuration.md +++ b/docs/src/content/configuration.md @@ -266,7 +266,7 @@ defaults: Tags are filtered through a pipeline: 1. **glob + semver**: build the candidate pool by intersecting the glob match set (default `*`) with the version range -2. **exclude**: remove tags matching any user `exclude` pattern OR any default-exclude pattern (see below) +2. **exclude**: remove tags matching any of three exclude tiers -- mapping `exclude:` (hard, blocks `include:`), `defaults.tags.exclude:` (soft, bypassable by `include:`), or the built-in prerelease list (soft, bypassable by `include:`) 3. **sort**: order the pool (`semver` or `alpha`) 4. **latest**: keep only the N most recent of the pool 5. **include**: union always-include tag matches into the result (not subject to `glob`, `semver`, default-excludes, `sort`, or `latest`); still subject to user `exclude` @@ -277,7 +277,7 @@ All filters are optional. Without any filters, all tags are synced. | Field | Type | Description | |---|---|---| | `glob` | string or list | Include tags matching glob pattern(s). A single string or a list of patterns | -| `include` | string or list | Always-include glob pattern(s). Tags matching any pattern survive `glob:`/`semver:` filters and the system-exclude defaults. Same syntax as `exclude:` | +| `include` | string or list | Always-include glob pattern(s). Tags matching any pattern survive `glob:`/`semver:` filters and the soft exclude tier (built-in defaults + `defaults.tags.exclude:`). Subject to mapping `exclude:` (hard tier). Same syntax as `exclude:` | | `semver` | string | Include tags satisfying a version range. Operators: `>=`, `<=`, `>`, `<`, `=`. Comma-joined for AND-narrowing. Example: `">=1.0, <2.0"` | | `exclude` | string or list | Remove tags matching these glob pattern(s) | | `sort` | string | Sort order for remaining tags: `semver` or `alpha` | @@ -292,11 +292,18 @@ All filters are optional. Without any filters, all tags are synced. `latest:` is optional, but mirrors with `semver:` and no `latest:` cap will sync every tag matching the version range. Under the lenient parser, popular images often publish hundreds of variant tags (`-alpine`, `-r0`, `-debian-12-rN`, `-bookworm-slim`, etc.). For long-running mirrors, set `latest: N` (with `sort: semver`) to cap output size. ocync emits a startup warning when `semver:` is set without `latest:`. -**Override semantics:** when a mapping defines `tags:`, the entire block replaces `defaults.tags` - fields are not merged. If you want a mapping to inherit some default fields and override others, repeat the inherited fields in the mapping's `tags:` block. +**Override semantics:** mapping fields override `defaults.tags` field by field. Any field unset on a mapping falls through to the corresponding field on `defaults.tags` (if present). The two `exclude` lists are kept on separate tiers, not merged into one: -### Default-exclude patterns +- `defaults.tags.exclude:` is the **soft tier**. It applies to every mapping that inherits from defaults, and `include:` on any mapping can override it. Use this for project-wide opinions ("drop `*-dev` unless I say otherwise"). +- `mapping.tags.exclude:` is the **hard tier**. It applies only to that mapping and is NOT overridden by `include:` on the same mapping. Use this for absolute per-mapping denies. -ocync drops common prerelease-marker tag patterns by default to keep mirrors focused on stable releases. The default-exclude list (case-insensitive) is: +Both tiers apply (concat). To rescue a tag the soft tier would drop on a specific mapping, add the tag to that mapping's `include:`. + +> **Behavior change.** Earlier versions replaced the whole `tags:` block when a mapping defined its own, so `defaults.tags.exclude:` was silently dropped for mappings with any per-mapping tag config. It now always applies (as the soft tier). If you were relying on `defaults.tags.exclude:` blocking an `include:` somewhere, switch that pattern to `mapping.tags.exclude:` (the hard tier, which still blocks `include:` on the same mapping). + +### Built-in exclude patterns + +ocync ships with a baked-in glob list that drops common prerelease-marker tag patterns to keep mirrors focused on stable releases. These patterns are part of the soft tier alongside `defaults.tags.exclude:`, so `include:` bypasses them. The list (case-insensitive) is: - `*-rc*` - `*-alpha*` @@ -305,14 +312,14 @@ ocync drops common prerelease-marker tag patterns by default to keep mirrors foc - `*-snapshot*` - `*-nightly*` -Patterns deliberately NOT in the default list (still admitted unless you exclude them yourself): +Patterns deliberately NOT in the built-in list (still admitted unless your `defaults.tags.exclude:` or `mapping.tags.exclude:` lists them): - `*-dev*` -- Chainguard publishes `latest-dev` and `1.25.5-dev` as stable variants - `*-edge*` -- Alpine rolling stable channel - `*-final*` -- Java stable marker - `-r` -- Chainguard/Bitnami build counters -To opt back into prereleases, add them to `include:` (which overrides the default-exclude). To pin a single prerelease tag for testing, add the exact tag string to `include:`. To add custom exclude patterns on top of the defaults, use `exclude:`. +To opt back into prereleases, add them to `include:` (which overrides the built-in list and `defaults.tags.exclude:`). To pin a single prerelease tag for testing, add the exact tag string to `include:`. To add project-wide exclude patterns, use `defaults.tags.exclude:`. To deny a tag absolutely on one mapping, use `mapping.tags.exclude:`. ## Environment variables @@ -410,15 +417,10 @@ mappings: - from: library/postgres to: postgres tags: - # Mapping tags: replaces defaults.tags entirely. Repeat the - # inherited fields explicitly when a mapping needs both. + # Mapping fields override defaults field by field. Unset fields + # (sort, min_tags, exclude soft tier) inherit from defaults.tags. semver: ">=15" - exclude: - - "*-debug" - - "*-rc*" - sort: semver latest: 3 - min_tags: 1 ``` ### GHCR to ECR with glob filtering diff --git a/src/cli/commands/dry_run.rs b/src/cli/commands/dry_run.rs index 69bd83f..e770980 100644 --- a/src/cli/commands/dry_run.rs +++ b/src/cli/commands/dry_run.rs @@ -164,10 +164,13 @@ fn write_dropped(w: &mut W, report: &FilterReport, verbose: bool) -> i " {:>4} {:<28}{}", reason.count, display_label, samples_display )?; - if matches!(reason.kind, DropKind::SystemExclude) { + if matches!( + reason.kind, + DropKind::BuiltinExclude | DropKind::DefaultsExclude { .. } + ) { writeln!( w, - " hint: to keep prereleases, list patterns under include: (globs supported)" + " hint: to keep these tags, list them under include: (globs supported)" )?; } } @@ -247,7 +250,7 @@ mod tests { ], }, DropReason { - kind: DropKind::SystemExclude, + kind: DropKind::BuiltinExclude, count: 2, samples: vec!["3.18.0-rc.1".into(), "3.19.0-beta.1".into()], }, @@ -301,7 +304,7 @@ mod tests { Ok(()) }); assert!( - out.contains("to keep prereleases, list patterns under include:"), + out.contains("hint: to keep these tags, list them under include:"), "{out}" ); } diff --git a/src/cli/commands/synchronize.rs b/src/cli/commands/synchronize.rs index 210c87b..04b1b63 100644 --- a/src/cli/commands/synchronize.rs +++ b/src/cli/commands/synchronize.rs @@ -713,16 +713,15 @@ pub(crate) async fn resolve_mapping( // --- Fetch and filter tags --- let source_repo_path = RepositoryName::new(&mapping.from)?; - let tags_config = mapping - .tags - .as_ref() - .or(config.defaults.as_ref().and_then(|d| d.tags.as_ref())); + let mapping_tags = mapping.tags.as_ref(); + let defaults_tags = config.defaults.as_ref().and_then(|d| d.tags.as_ref()); // Fast path: when the config specifies only exact tag names (no // wildcards, semver, latest, exclude), use them directly without - // enumerating all tags from the source registry. This avoids - // hundreds of paginated tags/list requests for repos with thousands - // of tags. + // enumerating all tags from the source registry. The fast path is + // gated on the mapping having no `defaults.tags` block in play -- + // otherwise inherited filters (notably `defaults.exclude`) would be + // skipped silently. // The image/artifact partition + sample collection happen in the same // pass that prepares input for `select_filtered_tags`, so the filter and // the no-match WARN both see consistent counts. The pre-built `NoTagsInfo` @@ -732,7 +731,10 @@ pub(crate) async fn resolve_mapping( Option, Option, Option, - ) = if let Some(exact) = tags_config.and_then(|t| t.exact_tags()) { + ) = if let Some(exact) = mapping_tags + .filter(|_| defaults_tags.is_none()) + .and_then(|t| t.exact_tags()) + { (exact, None, None, None) } else { let all_tags = source_client.list_tags(&source_repo_path).await?; @@ -750,10 +752,11 @@ pub(crate) async fn resolve_mapping( from: mapping.from.clone(), image_count, artifact_count: all_tags.len() - image_count, - filter_desc: describe_filter(tags_config), + filter_desc: describe_filter(mapping_tags, defaults_tags), samples, }; - let (kept, count, report) = select_filtered_tags(tags_config, all_tags, with_report)?; + let (kept, count, report) = + select_filtered_tags(mapping_tags, defaults_tags, all_tags, with_report)?; (kept, count, report, Some(template)) }; @@ -762,7 +765,7 @@ pub(crate) async fn resolve_mapping( from: mapping.from.clone(), image_count: 0, artifact_count: 0, - filter_desc: describe_filter(tags_config), + filter_desc: describe_filter(mapping_tags, defaults_tags), samples: Vec::new(), }); return Ok(MappingResolution::NoMatchingTags(info)); @@ -795,7 +798,7 @@ pub(crate) async fn resolve_mapping( .unwrap_or(false); // --- Immutable tags optimization --- - let immutable_pattern = tags_config.and_then(|t| t.immutable_tags.as_deref()); + let immutable_pattern = resolve_immutable_pattern(mapping_tags, defaults_tags); let immutable_glob = if let Some(pattern) = immutable_pattern { let glob_set = build_glob_set(&[pattern.to_owned()])?; @@ -865,29 +868,65 @@ fn emit_no_tags_warn(info: &NoTagsInfo) { ); } -/// Build a `FilterConfig` from a `TagsConfig`, falling back to defaults. -fn build_filter(tags: Option<&TagsConfig>) -> FilterConfig { - let Some(tags) = tags else { - return FilterConfig::default(); +/// Build a `FilterConfig` from a mapping `TagsConfig` plus an optional +/// `defaults.tags` block. Field-level merge: any field set on the mapping +/// wins; unset fields fall through to `defaults`. The exclude lists are +/// kept separate by source -- mapping exclude is the hard tier (blocks +/// `include:`), defaults exclude is the soft tier (bypassable by `include:`). +fn build_filter(mapping: Option<&TagsConfig>, defaults: Option<&TagsConfig>) -> FilterConfig { + let pick_glob = |get: fn(&TagsConfig) -> Option<&GlobOrList>| -> Vec { + mapping + .and_then(get) + .or_else(|| defaults.and_then(get)) + .map(glob_or_list_to_vec_owned) + .unwrap_or_default() }; FilterConfig { - include: glob_or_list_to_vec(tags.include.as_ref()), - glob: glob_or_list_to_vec(tags.glob.as_ref()), - semver: tags.semver.clone(), - exclude: glob_or_list_to_vec(tags.exclude.as_ref()), - sort: tags.sort, - latest: tags.latest, - min_tags: tags.min_tags, + include: pick_glob(|t| t.include.as_ref()), + glob: pick_glob(|t| t.glob.as_ref()), + semver: mapping + .and_then(|t| t.semver.clone()) + .or_else(|| defaults.and_then(|t| t.semver.clone())), + defaults_exclude: defaults + .and_then(|t| t.exclude.as_ref()) + .map(glob_or_list_to_vec_owned) + .unwrap_or_default(), + exclude: mapping + .and_then(|t| t.exclude.as_ref()) + .map(glob_or_list_to_vec_owned) + .unwrap_or_default(), + sort: mapping + .and_then(|t| t.sort) + .or_else(|| defaults.and_then(|t| t.sort)), + latest: mapping + .and_then(|t| t.latest) + .or_else(|| defaults.and_then(|t| t.latest)), + min_tags: mapping + .and_then(|t| t.min_tags) + .or_else(|| defaults.and_then(|t| t.min_tags)), } } -/// Flatten a `GlobOrList` into a `Vec`. -fn glob_or_list_to_vec(g: Option<&GlobOrList>) -> Vec { +/// Resolve `immutable_tags` from a mapping + defaults pair: mapping wins, +/// then falls through to defaults. Lives outside [`build_filter`] because +/// `immutable_tags` is consumed by the skip-optimization path, not the +/// filter pipeline. +fn resolve_immutable_pattern<'a>( + mapping: Option<&'a TagsConfig>, + defaults: Option<&'a TagsConfig>, +) -> Option<&'a str> { + mapping + .and_then(|t| t.immutable_tags.as_deref()) + .or_else(|| defaults.and_then(|t| t.immutable_tags.as_deref())) +} + +/// Flatten a [`GlobOrList`] into an owned `Vec`. Used by the +/// merge path which already holds a borrow. +fn glob_or_list_to_vec_owned(g: &GlobOrList) -> Vec { match g { - Some(GlobOrList::Single(s)) => vec![s.clone()], - Some(GlobOrList::List(v)) => v.clone(), - None => Vec::new(), + GlobOrList::Single(s) => vec![s.clone()], + GlobOrList::List(v) => v.clone(), } } @@ -912,12 +951,13 @@ type SelectionResult = ( /// Extracted from [`resolve_mapping`] so the report wire-up is testable /// without spinning up a registry mock. fn select_filtered_tags( - tags_config: Option<&TagsConfig>, + mapping_tags: Option<&TagsConfig>, + defaults_tags: Option<&TagsConfig>, all_tags: Vec, with_report: bool, ) -> Result { let n_candidates = all_tags.len(); - let filter = build_filter(tags_config); + let filter = build_filter(mapping_tags, defaults_tags); let tag_refs: Vec<&str> = all_tags.iter().map(String::as_str).collect(); if with_report { let result = filter.apply_with_report(&tag_refs)?; @@ -927,14 +967,17 @@ fn select_filtered_tags( } } -/// One-line summary of a [`TagsConfig`] suitable for log emission, e.g. -/// `semver >=1.0.0, latest=5`. Returns `None` when no filter applies. +/// One-line summary of mapping + defaults tags suitable for log emission, +/// e.g. `semver >=1.0.0, latest=5`. Returns `None` when no filter applies. /// /// Single source of truth: delegates to [`FilterConfig::describe`] after -/// the same conversion the engine uses, so dry-run stage labels and the +/// the same merge the engine uses, so dry-run stage labels and the /// no-tags-matched WARN rationale cannot drift. -fn describe_filter(tags: Option<&TagsConfig>) -> Option { - build_filter(tags).describe() +fn describe_filter( + mapping_tags: Option<&TagsConfig>, + defaults_tags: Option<&TagsConfig>, +) -> Option { + build_filter(mapping_tags, defaults_tags).describe() } /// Write sync output as JSON when `--json` is passed. @@ -1034,22 +1077,75 @@ mod tests { #[test] fn build_filter_none_returns_default() { - let filter = build_filter(None); + let filter = build_filter(None, None); assert!(filter.glob.is_empty()); assert!(filter.semver.is_none()); assert!(filter.exclude.is_empty()); + assert!(filter.defaults_exclude.is_empty()); assert!(filter.sort.is_none()); assert!(filter.latest.is_none()); assert!(filter.min_tags.is_none()); } + /// `defaults.tags.exclude:` reaches a mapping that has its own `tags:` + /// block. Today's `or` resolution would drop it. After the merge, the + /// patterns land in `FilterConfig.defaults_exclude` (the soft tier). + #[test] + fn build_filter_defaults_exclude_reaches_mapping_with_own_tags() { + let mapping = TagsConfig { + semver: Some(">=1.0".into()), + ..Default::default() + }; + let defaults = TagsConfig { + exclude: Some(GlobOrList::List(vec!["*-dev".into(), "*-r[0-9]*".into()])), + ..Default::default() + }; + let filter = build_filter(Some(&mapping), Some(&defaults)); + assert_eq!(filter.semver.as_deref(), Some(">=1.0")); + assert_eq!(filter.defaults_exclude, vec!["*-dev", "*-r[0-9]*"]); + assert!(filter.exclude.is_empty(), "mapping had no exclude"); + } + + /// `mapping.tags.exclude:` lands in the hard tier; `defaults.exclude` + /// lands in the soft tier. Both apply (concat semantics). + #[test] + fn build_filter_mapping_exclude_is_hard_tier() { + let mapping = TagsConfig { + exclude: Some(GlobOrList::Single("*-slim".into())), + ..Default::default() + }; + let defaults = TagsConfig { + exclude: Some(GlobOrList::Single("*-dev".into())), + ..Default::default() + }; + let filter = build_filter(Some(&mapping), Some(&defaults)); + assert_eq!(filter.exclude, vec!["*-slim"]); + assert_eq!(filter.defaults_exclude, vec!["*-dev"]); + } + + /// When mapping has no `tags:` block at all, defaults' filter fields + /// flow through. `defaults.exclude` still goes to the soft tier -- + /// the source decides the tier, not whether the mapping was set. + #[test] + fn build_filter_inherits_defaults_when_mapping_unset() { + let defaults = TagsConfig { + semver: Some(">=2.0".into()), + exclude: Some(GlobOrList::Single("*-dev".into())), + ..Default::default() + }; + let filter = build_filter(None, Some(&defaults)); + assert_eq!(filter.semver.as_deref(), Some(">=2.0")); + assert_eq!(filter.defaults_exclude, vec!["*-dev"]); + assert!(filter.exclude.is_empty()); + } + #[test] fn build_filter_single_glob() { let tags = TagsConfig { glob: Some(GlobOrList::Single("v1.*".into())), ..Default::default() }; - let filter = build_filter(Some(&tags)); + let filter = build_filter(Some(&tags), None); assert_eq!(filter.glob, vec!["v1.*"]); } @@ -1059,7 +1155,7 @@ mod tests { glob: Some(GlobOrList::List(vec!["v1.*".into(), "v2.*".into()])), ..Default::default() }; - let filter = build_filter(Some(&tags)); + let filter = build_filter(Some(&tags), None); assert_eq!(filter.glob, vec!["v1.*", "v2.*"]); } @@ -1069,7 +1165,7 @@ mod tests { exclude: Some(GlobOrList::List(vec!["*-rc*".into(), "*-beta*".into()])), ..Default::default() }; - let filter = build_filter(Some(&tags)); + let filter = build_filter(Some(&tags), None); assert_eq!(filter.exclude, vec!["*-rc*", "*-beta*"]); } @@ -1088,7 +1184,7 @@ mod tests { immutable_tags: None, ..Default::default() }; - let filter = build_filter(Some(&tags)); + let filter = build_filter(Some(&tags), None); assert_eq!(filter.include, vec!["latest"]); assert_eq!(filter.glob, vec!["*"]); assert_eq!(filter.semver.as_deref(), Some(">=1.0.0")); @@ -1110,7 +1206,7 @@ sort: semver latest: 5 "#; let tags: TagsConfig = serde_yaml::from_str(tags_yaml).expect("yaml parses"); - let filter = build_filter(Some(&tags)); + let filter = build_filter(Some(&tags), None); // Confirm the FilterConfig was built with the right include patterns. assert_eq!( @@ -1147,21 +1243,267 @@ latest: 5 assert_eq!(result.len(), 7); } + /// Every fall-through field on the merge model: `sort`, `latest`, + /// `min_tags`, `include`, `glob`, `semver`. When the mapping leaves + /// each unset, the value comes from `defaults.tags`; when set, the + /// mapping wins. One test, six pairs of assertions, no scaffolding. + #[test] + fn merge_inherits_all_fall_through_fields() { + use ocync_sync::filter::SortOrder; + + let defaults = TagsConfig { + include: Some(GlobOrList::Single("latest".into())), + glob: Some(GlobOrList::Single("v*".into())), + semver: Some(">=1.0".into()), + sort: Some(SortOrder::Semver), + latest: Some(10), + min_tags: Some(2), + immutable_tags: Some("v?[0-9]*.[0-9]*.[0-9]*".into()), + ..Default::default() + }; + + // 1. Mapping unset on everything: defaults flow through. + let empty_mapping = TagsConfig::default(); + let inherited = build_filter(Some(&empty_mapping), Some(&defaults)); + assert_eq!(inherited.include, vec!["latest"]); + assert_eq!(inherited.glob, vec!["v*"]); + assert_eq!(inherited.semver.as_deref(), Some(">=1.0")); + assert_eq!(inherited.sort, Some(SortOrder::Semver)); + assert_eq!(inherited.latest, Some(10)); + assert_eq!(inherited.min_tags, Some(2)); + assert_eq!( + resolve_immutable_pattern(Some(&empty_mapping), Some(&defaults)), + Some("v?[0-9]*.[0-9]*.[0-9]*"), + ); + + // 2. Mapping sets every field: mapping wins on every field. + let override_mapping = TagsConfig { + include: Some(GlobOrList::Single("override".into())), + glob: Some(GlobOrList::Single("override*".into())), + semver: Some(">=2.0".into()), + sort: Some(SortOrder::Alpha), + latest: Some(3), + min_tags: Some(1), + immutable_tags: Some("override-pattern".into()), + ..Default::default() + }; + let overridden = build_filter(Some(&override_mapping), Some(&defaults)); + assert_eq!(overridden.include, vec!["override"]); + assert_eq!(overridden.glob, vec!["override*"]); + assert_eq!(overridden.semver.as_deref(), Some(">=2.0")); + assert_eq!(overridden.sort, Some(SortOrder::Alpha)); + assert_eq!(overridden.latest, Some(3)); + assert_eq!(overridden.min_tags, Some(1)); + assert_eq!( + resolve_immutable_pattern(Some(&override_mapping), Some(&defaults)), + Some("override-pattern"), + ); + } + + /// `resolve_immutable_pattern` falls back when only one side carries + /// a pattern, and returns `None` when neither does. + #[test] + fn resolve_immutable_pattern_handles_partial_set() { + let with_immutable = TagsConfig { + immutable_tags: Some("v?[0-9]*".into()), + ..Default::default() + }; + let empty = TagsConfig::default(); + + assert_eq!( + resolve_immutable_pattern(Some(&with_immutable), None), + Some("v?[0-9]*") + ); + assert_eq!( + resolve_immutable_pattern(None, Some(&with_immutable)), + Some("v?[0-9]*") + ); + assert_eq!( + resolve_immutable_pattern(Some(&empty), Some(&with_immutable)), + Some("v?[0-9]*") + ); + assert_eq!(resolve_immutable_pattern(Some(&empty), Some(&empty)), None); + assert_eq!(resolve_immutable_pattern(None, None), None); + } + + /// Realistic Chainguard scenario: `defaults.exclude` filters dev and + /// `-rN` revisions across the project, one mapping uses `include:` to + /// rescue `latest-dev`, another adds a hard-tier mapping `exclude:` + /// for `-slim` variants. Asserts the final keep set + dry-run drop + /// attribution by tier. #[test] - fn glob_or_list_to_vec_none() { - assert!(glob_or_list_to_vec(None).is_empty()); + fn merge_chainguard_scenario_end_to_end() { + let defaults_yaml = r#" +exclude: ["*-dev", "*-r[0-9]*"] +"#; + let defaults: TagsConfig = + serde_yaml::from_str(defaults_yaml).expect("defaults yaml parses"); + + // Realistic cgr.dev tag list for a single repo: stable releases, + // dev variants, package revisions, slim variants, an RC. + let tags = vec![ + "1.27", + "1.27-r0", + "1.27-r1", + "1.27-dev", + "1.27-slim", + "1.27-rc1", + "latest", + "latest-dev", + ]; + + // --- Mapping A: pure inheritance (no `tags:` block) --- + // Today's bug: this mapping silently lost `defaults.exclude`. After + // the fix, all dev/-rN variants drop, RC drops via built-in. + let no_mapping_filter = build_filter(None, Some(&defaults)); + let kept_a = no_mapping_filter + .apply(&tags) + .expect("filter applies (mapping A)"); + assert_eq!( + kept_a, + vec![ + "1.27".to_string(), + "1.27-slim".to_string(), + "latest".to_string() + ], + "mapping A should keep stable + slim + latest only", + ); + + // --- Mapping B: `include: ["latest-dev"]` rescues from soft tier --- + let mapping_b: TagsConfig = serde_yaml::from_str( + r#" +include: ["latest-dev"] +"#, + ) + .expect("mapping B yaml parses"); + let filter_b = build_filter(Some(&mapping_b), Some(&defaults)); + let kept_b = filter_b.apply(&tags).expect("filter applies (mapping B)"); + assert!( + kept_b.contains(&"latest-dev".to_string()), + "include: should rescue latest-dev from defaults.exclude" + ); + assert!( + !kept_b.contains(&"1.27-dev".to_string()), + "non-included dev variants still drop" + ); + assert!( + !kept_b.contains(&"1.27-r0".to_string()), + "include: doesn't rescue what it doesn't list" + ); + + // --- Mapping C: hard-tier `exclude: ["*-slim"]` stacks on defaults --- + let mapping_c: TagsConfig = serde_yaml::from_str( + r#" +exclude: ["*-slim"] +"#, + ) + .expect("mapping C yaml parses"); + let filter_c = build_filter(Some(&mapping_c), Some(&defaults)); + let kept_c = filter_c.apply(&tags).expect("filter applies (mapping C)"); + assert_eq!( + kept_c, + vec!["1.27".to_string(), "latest".to_string()], + "mapping C drops slim (hard tier) + dev/-rN (soft tier) + rc (built-in)", + ); + + // --- Mapping D: dry-run attribution proves the tier breakdown --- + // Use mapping C's filter and verify each drop carries the right + // DropKind. This is the operator-facing observability check. + let report = filter_c + .apply_with_report(&tags) + .expect("apply_with_report succeeds"); + + let mapping_drops: Vec<&String> = report + .report + .dropped + .iter() + .filter(|d| matches!(d.kind, ocync_sync::filter::DropKind::MappingExclude { .. })) + .flat_map(|d| d.samples.iter()) + .collect(); + assert_eq!( + mapping_drops, + vec![&"1.27-slim".to_string()], + "MappingExclude bucket carries only the slim variant", + ); + + let defaults_drops: HashSet = report + .report + .dropped + .iter() + .filter(|d| matches!(d.kind, ocync_sync::filter::DropKind::DefaultsExclude { .. })) + .flat_map(|d| d.samples.iter().cloned()) + .collect(); + let expected_defaults: HashSet = ["1.27-r0", "1.27-r1", "1.27-dev", "latest-dev"] + .iter() + .map(|s| s.to_string()) + .collect(); + assert_eq!( + defaults_drops, expected_defaults, + "DefaultsExclude bucket carries dev + -rN variants" + ); + + let builtin_drops: Vec<&String> = report + .report + .dropped + .iter() + .filter(|d| matches!(d.kind, ocync_sync::filter::DropKind::BuiltinExclude)) + .flat_map(|d| d.samples.iter()) + .collect(); + assert_eq!( + builtin_drops, + vec![&"1.27-rc1".to_string()], + "BuiltinExclude bucket carries the RC tag" + ); + } + + /// YAML round-trip: parse a `defaults.tags` block + a per-mapping + /// `tags:` block via the production deserializer and confirm the merge + /// puts `defaults.exclude` patterns into the soft tier and mapping + /// patterns into the hard tier. Catches future serde-aliasing or + /// field-renaming regressions that bypass `build_filter`'s logic. + #[test] + fn merge_yaml_round_trip_separates_exclude_tiers() { + let defaults_yaml = r#" +exclude: ["*-dev", "*-r[0-9]*"] +sort: semver +latest: 5 +"#; + let mapping_yaml = r#" +semver: ">=1.0" +exclude: ["*-slim"] +"#; + let defaults: TagsConfig = + serde_yaml::from_str(defaults_yaml).expect("defaults yaml parses"); + let mapping: TagsConfig = serde_yaml::from_str(mapping_yaml).expect("mapping yaml parses"); + + let filter = build_filter(Some(&mapping), Some(&defaults)); + + // defaults.exclude reaches the soft tier verbatim. + assert_eq!(filter.defaults_exclude, vec!["*-dev", "*-r[0-9]*"]); + // mapping.exclude is the hard tier, separate from defaults. + assert_eq!(filter.exclude, vec!["*-slim"]); + // mapping fields override; unset fields inherit from defaults. + assert_eq!(filter.semver.as_deref(), Some(">=1.0")); + assert_eq!(filter.sort, Some(ocync_sync::filter::SortOrder::Semver)); + assert_eq!(filter.latest, Some(5)); + + // End-to-end behavior: 1.27-r0 dropped by defaults soft tier, + // 1.27-slim dropped by mapping hard tier, 1.27 survives. + let tags = vec!["1.27", "1.27-r0", "1.27-dev", "1.27-slim"]; + let kept = filter.apply(&tags).expect("filter applies"); + assert_eq!(kept, vec!["1.27".to_string()]); } #[test] - fn glob_or_list_to_vec_single() { + fn glob_or_list_to_vec_owned_single() { let g = GlobOrList::Single("pattern".into()); - assert_eq!(glob_or_list_to_vec(Some(&g)), vec!["pattern"]); + assert_eq!(glob_or_list_to_vec_owned(&g), vec!["pattern"]); } #[test] - fn glob_or_list_to_vec_list() { + fn glob_or_list_to_vec_owned_list() { let g = GlobOrList::List(vec!["a".into(), "b".into()]); - assert_eq!(glob_or_list_to_vec(Some(&g)), vec!["a", "b"]); + assert_eq!(glob_or_list_to_vec_owned(&g), vec!["a", "b"]); } // - parse_size ----------------------------------------------------------- @@ -1217,7 +1559,7 @@ latest: 5 ..TagsConfig::default() }; assert_eq!( - describe_filter(Some(&tags)).as_deref(), + describe_filter(Some(&tags), None).as_deref(), Some("semver >=1.0.0, latest=5") ); } @@ -1225,8 +1567,8 @@ latest: 5 #[test] fn describe_filter_returns_none_when_empty() { let tags = TagsConfig::default(); - assert!(describe_filter(Some(&tags)).is_none()); - assert!(describe_filter(None).is_none()); + assert!(describe_filter(Some(&tags), None).is_none()); + assert!(describe_filter(None, None).is_none()); } // -- NoTagsInfo Display --------------------------------------------- @@ -1587,7 +1929,7 @@ latest: 5 "0.9.0-rc1".into(), ]; let (kept, candidate_count, report) = - select_filtered_tags(Some(&tags_config), all_tags, true).unwrap(); + select_filtered_tags(Some(&tags_config), None, all_tags, true).unwrap(); // Wire-up: candidate count flows through. assert_eq!(candidate_count, Some(5)); @@ -1614,7 +1956,7 @@ latest: 5 }; let all_tags = vec!["1.0".into(), "2.0".into()]; let (kept, candidate_count, report) = - select_filtered_tags(Some(&tags_config), all_tags, false).unwrap(); + select_filtered_tags(Some(&tags_config), None, all_tags, false).unwrap(); assert_eq!(candidate_count, Some(2)); assert_eq!(kept.len(), 2); assert!(report.is_none()); @@ -1630,7 +1972,7 @@ latest: 5 ..Default::default() }; let all_tags = vec!["1.0".into(), "2.0".into()]; - let result = select_filtered_tags(Some(&tags_config), all_tags, false); + let result = select_filtered_tags(Some(&tags_config), None, all_tags, false); assert!( result.is_err(), "expected BelowMinTags error from real-sync path" @@ -1647,7 +1989,7 @@ latest: 5 }; let all_tags = vec!["1.0".into(), "2.0".into()]; let (kept, candidate_count, report) = - select_filtered_tags(Some(&tags_config), all_tags, true).unwrap(); + select_filtered_tags(Some(&tags_config), None, all_tags, true).unwrap(); assert_eq!(kept.len(), 2); assert_eq!(candidate_count, Some(2)); let report = report.expect("dry-run path returns report even when min_tags would error"); @@ -1671,7 +2013,7 @@ latest: 5 }; let all_tags: Vec = (0..10).map(|i| format!("1.{i}.0")).collect(); let (kept, candidate_count, filter_report) = - select_filtered_tags(Some(&tags_config), all_tags, true).unwrap(); + select_filtered_tags(Some(&tags_config), None, all_tags, true).unwrap(); assert_eq!(kept.len(), 0); // all dropped by semver >=2.0 assert!(filter_report.is_some());