Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,17 @@ released from this line yet.

## Diagnostics

* The printed `summary()` of a `bgm()` or `bgmCompare()` fit no longer shows
the directional flip counts `n0->1` and `n1->0` in its inclusion table. The
columns stay in `summary(fit)$indicator`.

* The printed `verdicts()` table closes with the number of verdicts that appear
robust to Monte Carlo error, followed by the number flagged because
their estimated log Bayes factors lie within two standard errors of an
evidence threshold. The text replaces the "Monte-Carlo fragile" warning,
which appeared only when some verdict was flagged, and suggests that a longer
run may help resolve the uncertainty. The `fragile` column is unchanged.

* The `summary()` tables and `prior_sensitivity_check()` no longer substitute
values for a Monte Carlo error or a convergence diagnostic that was not
computed. An indicator whose inclusion probability was 0 or 1 in every draw
Expand Down
7 changes: 4 additions & 3 deletions R/methods_bgmcompare.R
Original file line number Diff line number Diff line change
Expand Up @@ -327,10 +327,11 @@ print.summary.bgmCompare = function(x, digits = 3, ...) {

if(!is.null(x$indicator)) {
cat("Inclusion probabilities:\n")
# mean/mcse/sd/n_eff/Rhat are the Rao-Blackwellized inclusion estimate;
# n0->1 / n1->0 are the raw directional flip counts, which record the
# indicator's exploration beside them.
# mean/mcse/sd/n_eff/Rhat are the Rao-Blackwellized inclusion estimate. The
# directional flip counts n0->1 / n1->0 stay in summary(fit)$indicator and
# are not printed.
ind = head(x$indicator, 6)
ind = ind[, setdiff(names(ind), c("n0->1", "n1->0")), drop = FALSE]

ind_has_na = anyNA(ind)

Expand Down
7 changes: 4 additions & 3 deletions R/methods_bgms.R
Original file line number Diff line number Diff line change
Expand Up @@ -179,10 +179,11 @@ print.summary.bgms = function(x, digits = 3, ...) {

if(!is.null(x$indicator)) {
cat("Inclusion probabilities:\n")
# mean/mcse/sd/n_eff/Rhat are the Rao-Blackwellized inclusion estimate;
# n0->1 / n1->0 are the raw directional flip counts, which record the
# indicator's exploration beside them.
# mean/mcse/sd/n_eff/Rhat are the Rao-Blackwellized inclusion estimate. The
# directional flip counts n0->1 / n1->0 stay in summary(fit)$indicator and
# are not printed.
ind = head(x$indicator, .summary_preview_rows)
ind = ind[, setdiff(names(ind), c("n0->1", "n1->0")), drop = FALSE]
ind_has_na = anyNA(ind)
ind[] = lapply(ind, function(col) ifelse(is.na(col), "", round(col, digits)))
print(ind)
Expand Down
34 changes: 26 additions & 8 deletions R/verdicts.R
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,8 @@ build_verdicts = function(parameter, log_bf, pip, mcse, draws, evidence_threshol
#' where the Rao-Blackwellized draws are constant to double precision.}
#' \item{distance_two_state, distance_rb}{Distance from `log_bf` to the
#' nearer verdict boundary, in units of each standard error.}
#' \item{fragile}{`TRUE` when either distance is below 2.}
#' \item{fragile}{`TRUE` when either distance is below 2: the
#' near-threshold flag.}
#' }
#' The evidence threshold is attached as the `evidence_threshold` attribute,
#' and whether the fragility flag's operating point covers this kind of
Expand Down Expand Up @@ -442,7 +443,10 @@ check_evidence_threshold = function(evidence_threshold) {
#'
#' @description
#' Prints the verdict tally, the leading rows of the verdict table, and a
#' warning line when any verdict is Monte-Carlo fragile.
#' line giving the number of verdicts that appear robust to Monte Carlo
#' error, followed by the number flagged because their estimated log Bayes
#' factors lie within two standard errors of an evidence threshold (the
#' `fragile` column).
#'
#' @param x An object of class `bgms_verdicts`, from [verdicts()].
#' @param digits Number of digits for the printed numeric columns. Default `3`.
Expand Down Expand Up @@ -519,12 +523,26 @@ print.bgms_verdicts = function(x, digits = 3, max_rows = 10L, ...) {
cat(sprintf("... (%d more rows)\n", nrow(rows) - max_rows))
}

n_fragile = sum(rows$fragile)
if(n_fragile > 0) {
cat(sprintf(
"\n%d %s Monte-Carlo fragile: a verdict boundary lies within two standard\nerrors of the evidence, so the verdict could change on a rerun. Consider a\nlonger run.\n",
n_fragile, if(n_fragile == 1L) "verdict is" else "verdicts are"
))
# The unflagged count first, then the near-threshold count, over the
# indicators that carry a verdict.
fragile = rows$fragile %in% TRUE
n_robust = sum(!fragile & !is.na(rows$verdict))
n_fragile = sum(fragile)
if(n_robust + n_fragile > 0) {
first = sprintf(
"%d %s robust to Monte Carlo error.",
n_robust, if(n_robust == 1L) "verdict appears" else "verdicts appear"
)
second = if(n_fragile == 0L) {
"None of the estimated log Bayes factors is flagged as being within two standard errors of an evidence threshold."
} else {
sprintf(
"%d %s could change on a rerun: %s within two standard errors of an evidence threshold. A longer run may help resolve this uncertainty.",
n_fragile, if(n_fragile == 1L) "verdict" else "verdicts",
if(n_fragile == 1L) "its estimated log Bayes factor lies" else "their estimated log Bayes factors lie"
)
}
cat("\n", paste(strwrap(paste(first, second), width = 80), collapse = "\n"), "\n", sep = "")
}
# The flag's operating point was established on single-network edge
# indicators. Borrowing those numbers for difference indicators would report a
Expand Down
5 changes: 4 additions & 1 deletion man/print.bgms_verdicts.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion man/verdicts.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/testthat/_snaps/summary-marking.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
B-C

Inclusion probabilities:
parameter mean mcse sd n_eff Rhat n0->1 n1->0
parameter mean mcse sd n_eff Rhat
A (main)
A-B (pairwise)
A-C (pairwise)
Expand Down
12 changes: 6 additions & 6 deletions tests/testthat/_snaps/verdicts.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
A-C (pairwise) 0.10 -2.20 undecided TRUE
B-C (pairwise) 0.52 0.08 undecided FALSE

1 verdict is Monte-Carlo fragile: a verdict boundary lies within two standard
errors of the evidence, so the verdict could change on a rerun. Consider a
longer run.
2 verdicts appear robust to Monte Carlo error. 1 verdict could change on a
rerun: its estimated log Bayes factor lies within two standard errors of an
evidence threshold. A longer run may help resolve this uncertainty.

The fragility flag is not validated for difference indicators: its
operating point was established on single-network edge indicators only.
Expand Down Expand Up @@ -46,9 +46,9 @@
B-C (pairwise) 0.52 0.08 undecided FALSE
C (main) 0.50 0.00 undecided FALSE

1 verdict is Monte-Carlo fragile: a verdict boundary lies within two standard
errors of the evidence, so the verdict could change on a rerun. Consider a
longer run.
5 verdicts appear robust to Monte Carlo error. 1 verdict could change on a
rerun: its estimated log Bayes factor lies within two standard errors of an
evidence threshold. A longer run may help resolve this uncertainty.

The fragility flag is not validated for difference indicators: its
operating point was established on single-network edge indicators only.
Expand Down
22 changes: 13 additions & 9 deletions tests/testthat/test-verdicts.R
Original file line number Diff line number Diff line change
Expand Up @@ -233,21 +233,25 @@ test_that("print.bgms_verdicts tallies verdicts and warns once when fragile", {
fit = get_bgms_fit_wenchuan6()
v = verdicts(fit)

out = paste(utils::capture.output(print(v)), collapse = "\n")
out = gsub("\\s+", " ", paste(utils::capture.output(print(v)), collapse = " "))
expect_match(out, "presence \\d+ \\| undecided \\d+ \\| absence \\d+")
expect_match(out, "15 indicators")
# Edge indicators are what the flag was calibrated on, so no caveat here.
expect_false(grepl("not validated", out))
# The unflagged count comes first; the near-threshold count follows it.
expect_match(out, "appear robust to Monte Carlo error")
if(any(v$fragile)) {
expect_match(out, "Monte-Carlo fragile")
expect_match(out, "Consider a")
expect_match(out, sprintf("%d verdicts? could change on a rerun", sum(v$fragile)))
expect_match(out, "A longer run may help resolve this uncertainty")
}

# No fragile edges, no advice line.
# No fragile edges: every verdict is counted as unflagged.
v_none = v
v_none$fragile = rep(FALSE, nrow(v))
quiet = paste(utils::capture.output(print(v_none)), collapse = "\n")
expect_false(grepl("Monte-Carlo fragile", quiet))
quiet = gsub("\\s+", " ", paste(utils::capture.output(print(v_none)), collapse = " "))
expect_match(quiet, sprintf("%d verdicts appear robust", sum(!is.na(v$verdict))))
expect_match(quiet, "None of the estimated log Bayes factors is flagged as being within two standard errors of an evidence threshold")
expect_false(grepl("longer run", quiet))

# Selecting columns keeps the class but not the table; printing what is left
# must degrade to the plain data frame rather than fail on a missing column.
Expand Down Expand Up @@ -398,8 +402,8 @@ test_that("the compare print tabulates selected main differences as rows", {
expect_snapshot(print(make_compare_verdicts(main_selected = TRUE)))
})

test_that("the fragility footer suggests a longer run without commanding one", {
out = capture.output(print(make_compare_verdicts(main_selected = FALSE)))
expect_true(any(grepl("Consider a", out)))
test_that("the fragility footer suggests a longer run without promising resolution", {
out = gsub("\\s+", " ", paste(capture.output(print(make_compare_verdicts(main_selected = FALSE))), collapse = " "))
expect_true(any(grepl("A longer run may help resolve this uncertainty", out)))
expect_false(any(grepl("Run longer", out)))
})
Loading