diff --git a/NEWS.md b/NEWS.md index ae5edc5c..5c86bb56 100644 --- a/NEWS.md +++ b/NEWS.md @@ -165,6 +165,17 @@ released from this line yet. ## Diagnostics +* The printed `summary()` of a `bgm()` or `bgmCompare()` fit no longer shows + the directional flip counts `n0->1` and `n1->0` in its inclusion table. The + columns stay in `summary(fit)$indicator`. + +* The printed `verdicts()` table closes with the number of verdicts that appear + robust to Monte Carlo error, followed by the number flagged because + their estimated log Bayes factors lie within two standard errors of an + evidence threshold. The text replaces the "Monte-Carlo fragile" warning, + which appeared only when some verdict was flagged, and suggests that a longer + run may help resolve the uncertainty. The `fragile` column is unchanged. + * The `summary()` tables and `prior_sensitivity_check()` no longer substitute values for a Monte Carlo error or a convergence diagnostic that was not computed. An indicator whose inclusion probability was 0 or 1 in every draw diff --git a/R/methods_bgmcompare.R b/R/methods_bgmcompare.R index 0cb438c9..aec82921 100644 --- a/R/methods_bgmcompare.R +++ b/R/methods_bgmcompare.R @@ -327,10 +327,11 @@ print.summary.bgmCompare = function(x, digits = 3, ...) { if(!is.null(x$indicator)) { cat("Inclusion probabilities:\n") - # mean/mcse/sd/n_eff/Rhat are the Rao-Blackwellized inclusion estimate; - # n0->1 / n1->0 are the raw directional flip counts, which record the - # indicator's exploration beside them. + # mean/mcse/sd/n_eff/Rhat are the Rao-Blackwellized inclusion estimate. The + # directional flip counts n0->1 / n1->0 stay in summary(fit)$indicator and + # are not printed. ind = head(x$indicator, 6) + ind = ind[, setdiff(names(ind), c("n0->1", "n1->0")), drop = FALSE] ind_has_na = anyNA(ind) diff --git a/R/methods_bgms.R b/R/methods_bgms.R index 520d890d..e3d86efe 100644 --- a/R/methods_bgms.R +++ b/R/methods_bgms.R @@ -179,10 +179,11 @@ print.summary.bgms = function(x, digits = 3, ...) { if(!is.null(x$indicator)) { cat("Inclusion probabilities:\n") - # mean/mcse/sd/n_eff/Rhat are the Rao-Blackwellized inclusion estimate; - # n0->1 / n1->0 are the raw directional flip counts, which record the - # indicator's exploration beside them. + # mean/mcse/sd/n_eff/Rhat are the Rao-Blackwellized inclusion estimate. The + # directional flip counts n0->1 / n1->0 stay in summary(fit)$indicator and + # are not printed. ind = head(x$indicator, .summary_preview_rows) + ind = ind[, setdiff(names(ind), c("n0->1", "n1->0")), drop = FALSE] ind_has_na = anyNA(ind) ind[] = lapply(ind, function(col) ifelse(is.na(col), "", round(col, digits))) print(ind) diff --git a/R/verdicts.R b/R/verdicts.R index f728c65f..098c3c7c 100644 --- a/R/verdicts.R +++ b/R/verdicts.R @@ -275,7 +275,8 @@ build_verdicts = function(parameter, log_bf, pip, mcse, draws, evidence_threshol #' where the Rao-Blackwellized draws are constant to double precision.} #' \item{distance_two_state, distance_rb}{Distance from `log_bf` to the #' nearer verdict boundary, in units of each standard error.} -#' \item{fragile}{`TRUE` when either distance is below 2.} +#' \item{fragile}{`TRUE` when either distance is below 2: the +#' near-threshold flag.} #' } #' The evidence threshold is attached as the `evidence_threshold` attribute, #' and whether the fragility flag's operating point covers this kind of @@ -442,7 +443,10 @@ check_evidence_threshold = function(evidence_threshold) { #' #' @description #' Prints the verdict tally, the leading rows of the verdict table, and a -#' warning line when any verdict is Monte-Carlo fragile. +#' line giving the number of verdicts that appear robust to Monte Carlo +#' error, followed by the number flagged because their estimated log Bayes +#' factors lie within two standard errors of an evidence threshold (the +#' `fragile` column). #' #' @param x An object of class `bgms_verdicts`, from [verdicts()]. #' @param digits Number of digits for the printed numeric columns. Default `3`. @@ -519,12 +523,26 @@ print.bgms_verdicts = function(x, digits = 3, max_rows = 10L, ...) { cat(sprintf("... (%d more rows)\n", nrow(rows) - max_rows)) } - n_fragile = sum(rows$fragile) - if(n_fragile > 0) { - cat(sprintf( - "\n%d %s Monte-Carlo fragile: a verdict boundary lies within two standard\nerrors of the evidence, so the verdict could change on a rerun. Consider a\nlonger run.\n", - n_fragile, if(n_fragile == 1L) "verdict is" else "verdicts are" - )) + # The unflagged count first, then the near-threshold count, over the + # indicators that carry a verdict. + fragile = rows$fragile %in% TRUE + n_robust = sum(!fragile & !is.na(rows$verdict)) + n_fragile = sum(fragile) + if(n_robust + n_fragile > 0) { + first = sprintf( + "%d %s robust to Monte Carlo error.", + n_robust, if(n_robust == 1L) "verdict appears" else "verdicts appear" + ) + second = if(n_fragile == 0L) { + "None of the estimated log Bayes factors is flagged as being within two standard errors of an evidence threshold." + } else { + sprintf( + "%d %s could change on a rerun: %s within two standard errors of an evidence threshold. A longer run may help resolve this uncertainty.", + n_fragile, if(n_fragile == 1L) "verdict" else "verdicts", + if(n_fragile == 1L) "its estimated log Bayes factor lies" else "their estimated log Bayes factors lie" + ) + } + cat("\n", paste(strwrap(paste(first, second), width = 80), collapse = "\n"), "\n", sep = "") } # The flag's operating point was established on single-network edge # indicators. Borrowing those numbers for difference indicators would report a diff --git a/man/print.bgms_verdicts.Rd b/man/print.bgms_verdicts.Rd index 3266781d..f10977e0 100644 --- a/man/print.bgms_verdicts.Rd +++ b/man/print.bgms_verdicts.Rd @@ -20,7 +20,10 @@ } \description{ Prints the verdict tally, the leading rows of the verdict table, and a -warning line when any verdict is Monte-Carlo fragile. +line giving the number of verdicts that appear robust to Monte Carlo +error, followed by the number flagged because their estimated log Bayes +factors lie within two standard errors of an evidence threshold (the +\code{fragile} column). } \seealso{ \code{\link[=verdicts]{verdicts()}} diff --git a/man/verdicts.Rd b/man/verdicts.Rd index 66370809..27f2df9d 100644 --- a/man/verdicts.Rd +++ b/man/verdicts.Rd @@ -36,7 +36,8 @@ indicator chain and from the Rao-Blackwellized draws. \code{se_rb} is \code{NA} where the Rao-Blackwellized draws are constant to double precision.} \item{distance_two_state, distance_rb}{Distance from \code{log_bf} to the nearer verdict boundary, in units of each standard error.} -\item{fragile}{\code{TRUE} when either distance is below 2.} +\item{fragile}{\code{TRUE} when either distance is below 2: the +near-threshold flag.} } The evidence threshold is attached as the \code{evidence_threshold} attribute, and whether the fragility flag's operating point covers this kind of diff --git a/tests/testthat/_snaps/summary-marking.md b/tests/testthat/_snaps/summary-marking.md index f288cedf..e666f460 100644 --- a/tests/testthat/_snaps/summary-marking.md +++ b/tests/testthat/_snaps/summary-marking.md @@ -24,7 +24,7 @@ B-C Inclusion probabilities: - parameter mean mcse sd n_eff Rhat n0->1 n1->0 + parameter mean mcse sd n_eff Rhat A (main) A-B (pairwise) A-C (pairwise) diff --git a/tests/testthat/_snaps/verdicts.md b/tests/testthat/_snaps/verdicts.md index 24f3c358..63599099 100644 --- a/tests/testthat/_snaps/verdicts.md +++ b/tests/testthat/_snaps/verdicts.md @@ -14,9 +14,9 @@ A-C (pairwise) 0.10 -2.20 undecided TRUE B-C (pairwise) 0.52 0.08 undecided FALSE - 1 verdict is Monte-Carlo fragile: a verdict boundary lies within two standard - errors of the evidence, so the verdict could change on a rerun. Consider a - longer run. + 2 verdicts appear robust to Monte Carlo error. 1 verdict could change on a + rerun: its estimated log Bayes factor lies within two standard errors of an + evidence threshold. A longer run may help resolve this uncertainty. The fragility flag is not validated for difference indicators: its operating point was established on single-network edge indicators only. @@ -46,9 +46,9 @@ B-C (pairwise) 0.52 0.08 undecided FALSE C (main) 0.50 0.00 undecided FALSE - 1 verdict is Monte-Carlo fragile: a verdict boundary lies within two standard - errors of the evidence, so the verdict could change on a rerun. Consider a - longer run. + 5 verdicts appear robust to Monte Carlo error. 1 verdict could change on a + rerun: its estimated log Bayes factor lies within two standard errors of an + evidence threshold. A longer run may help resolve this uncertainty. The fragility flag is not validated for difference indicators: its operating point was established on single-network edge indicators only. diff --git a/tests/testthat/test-verdicts.R b/tests/testthat/test-verdicts.R index a85de48a..0fca88a7 100644 --- a/tests/testthat/test-verdicts.R +++ b/tests/testthat/test-verdicts.R @@ -233,21 +233,25 @@ test_that("print.bgms_verdicts tallies verdicts and warns once when fragile", { fit = get_bgms_fit_wenchuan6() v = verdicts(fit) - out = paste(utils::capture.output(print(v)), collapse = "\n") + out = gsub("\\s+", " ", paste(utils::capture.output(print(v)), collapse = " ")) expect_match(out, "presence \\d+ \\| undecided \\d+ \\| absence \\d+") expect_match(out, "15 indicators") # Edge indicators are what the flag was calibrated on, so no caveat here. expect_false(grepl("not validated", out)) + # The unflagged count comes first; the near-threshold count follows it. + expect_match(out, "appear robust to Monte Carlo error") if(any(v$fragile)) { - expect_match(out, "Monte-Carlo fragile") - expect_match(out, "Consider a") + expect_match(out, sprintf("%d verdicts? could change on a rerun", sum(v$fragile))) + expect_match(out, "A longer run may help resolve this uncertainty") } - # No fragile edges, no advice line. + # No fragile edges: every verdict is counted as unflagged. v_none = v v_none$fragile = rep(FALSE, nrow(v)) - quiet = paste(utils::capture.output(print(v_none)), collapse = "\n") - expect_false(grepl("Monte-Carlo fragile", quiet)) + quiet = gsub("\\s+", " ", paste(utils::capture.output(print(v_none)), collapse = " ")) + expect_match(quiet, sprintf("%d verdicts appear robust", sum(!is.na(v$verdict)))) + expect_match(quiet, "None of the estimated log Bayes factors is flagged as being within two standard errors of an evidence threshold") + expect_false(grepl("longer run", quiet)) # Selecting columns keeps the class but not the table; printing what is left # must degrade to the plain data frame rather than fail on a missing column. @@ -398,8 +402,8 @@ test_that("the compare print tabulates selected main differences as rows", { expect_snapshot(print(make_compare_verdicts(main_selected = TRUE))) }) -test_that("the fragility footer suggests a longer run without commanding one", { - out = capture.output(print(make_compare_verdicts(main_selected = FALSE))) - expect_true(any(grepl("Consider a", out))) +test_that("the fragility footer suggests a longer run without promising resolution", { + out = gsub("\\s+", " ", paste(capture.output(print(make_compare_verdicts(main_selected = FALSE))), collapse = " ")) + expect_true(any(grepl("A longer run may help resolve this uncertainty", out))) expect_false(any(grepl("Run longer", out))) })