From 50cdcd1a63f203059977213020d01109d0db862c Mon Sep 17 00:00:00 2001 From: squid-protocol Date: Thu, 8 Oct 2026 03:26:14 -0400 Subject: [PATCH 1/2] det: escape every control/high byte in Java literals (#4690); DISPLAY writes record-charset bytes (#4691) jstr emits three-digit octal escapes for 0x00-0x1F and 0x7F-0xFF (no u-escapes: javac translates u000a/u000d before lexing). Sysout (batch forge and standalone) encodes with the record charset instead of UTF-8; equivalence.compare_sysout decodes the port's SYSOUT with the data encoding. Co-Authored-By: Claude Opus 5.5 --- .../cobol_to_java_batch_forge.py | 18 +++--- .../det/cobolrt/standalone/Sysout.java | 27 ++++++++- gitgalaxy/tools/cobol_to_java/det/gen.py | 4 ++ tests/cobol_mainframe/test_det_bytes_4690.py | 55 +++++++++++++++++++ tests/cobol_mainframe/test_det_programs.py | 15 +++-- tests/tools/equivalence.py | 5 +- 6 files changed, 109 insertions(+), 15 deletions(-) create mode 100644 tests/cobol_mainframe/test_det_bytes_4690.py diff --git a/gitgalaxy/tools/cobol_to_java/cobol_to_java_batch_forge.py b/gitgalaxy/tools/cobol_to_java/cobol_to_java_batch_forge.py index 8c4d4eeb5c..0b610f48e0 100644 --- a/gitgalaxy/tools/cobol_to_java/cobol_to_java_batch_forge.py +++ b/gitgalaxy/tools/cobol_to_java/cobol_to_java_batch_forge.py @@ -45,7 +45,7 @@ from gitgalaxy.tools.cobol_to_java.cobol_to_java_common import TraceLog, java_path, status_text from gitgalaxy.tools.cobol_to_java.cobol_to_java_names import java_class_base -from gitgalaxy.tools.cobol_to_java.java_target import JavaTarget +from gitgalaxy.tools.cobol_to_java.java_target import JavaTarget, record_charset_java SUBPACKAGE = "batch" # IBM's PL/I compiler, spelled in two parts: its name holds the two-byte DOS executable @@ -222,7 +222,9 @@ def sources(self) -> dict[tuple[str, ...], dict[str, str]]: return {} pkg = f"{self.package}.{SUBPACKAGE}" out = { - name: text.replace("{pkg}", pkg).replace("{zone}", self.target.culture.zone) + name: text.replace("{pkg}", pkg) + .replace("{zone}", self.target.culture.zone) + .replace("{record_charset}", record_charset_java(self.target.data.record_charset)) for name, text in _RUNTIME.items() } if self.target.features.rest_controllers: @@ -407,7 +409,7 @@ def audit_line(self) -> str: import java.io.UncheckedIOException; import java.math.BigDecimal; import java.math.RoundingMode; -import java.nio.charset.StandardCharsets; +import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -416,7 +418,9 @@ def audit_line(self) -> str: * The job log (#4056): a COBOL `DISPLAY` is `Sysout.display(...)` -- one line, its operands' text concatenated * exactly as IBM Enterprise COBOL writes them -- not a log line. SYSOUT is what operators read and scripts parse, * and the equivalence harness compares it line by line (trailing blanks aside). Written to standard output, or - * appended to the file named by the system property gitgalaxy.sysout. + * appended to the file named by the system property gitgalaxy.sysout. #4691: the bytes are written in the record + * charset (system property gitgalaxy.data.charset, else the target's data.record_charset), never UTF-8: a byte + * above X'7F' leaves as that one byte, as the COBOL runtime writes it. */ public final class Sysout { @@ -464,15 +468,15 @@ def audit_line(self) -> str: private static void write(String text) { String target = System.getProperty("gitgalaxy.sysout"); + byte[] bytes = text.getBytes(Charset.forName(System.getProperty("gitgalaxy.data.charset", "{record_charset}"))); synchronized (LOCK) { if (target == null) { - System.out.print(text); + System.out.write(bytes, 0, bytes.length); System.out.flush(); return; } try { - Files.writeString(Path.of(target), text, StandardCharsets.UTF_8, StandardOpenOption.CREATE, - StandardOpenOption.APPEND); + Files.write(Path.of(target), bytes, StandardOpenOption.CREATE, StandardOpenOption.APPEND); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/gitgalaxy/tools/cobol_to_java/det/cobolrt/standalone/Sysout.java b/gitgalaxy/tools/cobol_to_java/det/cobolrt/standalone/Sysout.java index 909012123a..984363ba8c 100644 --- a/gitgalaxy/tools/cobol_to_java/det/cobolrt/standalone/Sysout.java +++ b/gitgalaxy/tools/cobol_to_java/det/cobolrt/standalone/Sysout.java @@ -1,16 +1,22 @@ package __PACKAGE__.cobolrt.standalone; -/** DISPLAY for a project the generator gave no batch package (a CICS-only estate): the operands' texts, one line. */ +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/** + * DISPLAY for a project the generator gave no batch package (a CICS-only estate): the operands' texts, one line. + * #4691: written in the record charset, not UTF-8, so a byte above X'7F' leaves as that one byte. + */ public final class Sysout { private Sysout() { } public static void display(Object... operands) { - System.out.println(join(operands)); + write(join(operands) + "\n"); } public static void displayNoAdvancing(Object... operands) { - System.out.print(join(operands)); + write(join(operands)); } private static String join(Object... operands) { @@ -20,4 +26,19 @@ private static String join(Object... operands) { } return b.toString(); } + + /** The record charset, CobolRecords.charset() when the project has it (looked up, so this class stands alone). */ + private static Charset charset() { + try { + return (Charset) Class.forName("__PACKAGE__.entity.vsam.CobolRecords").getMethod("charset").invoke(null); + } catch (ReflectiveOperationException | ClassCastException e) { + return StandardCharsets.ISO_8859_1; + } + } + + private static void write(String text) { + byte[] bytes = text.getBytes(charset()); + System.out.write(bytes, 0, bytes.length); + System.out.flush(); + } } diff --git a/gitgalaxy/tools/cobol_to_java/det/gen.py b/gitgalaxy/tools/cobol_to_java/det/gen.py index a8c8502451..c5e12dbfb7 100644 --- a/gitgalaxy/tools/cobol_to_java/det/gen.py +++ b/gitgalaxy/tools/cobol_to_java/det/gen.py @@ -67,6 +67,10 @@ def jstr(s: str) -> str: out.append("\\\\") elif 32 <= o < 127: out.append(ch) + elif o < 256: + # #4690: never the u-escapes 000a / 000d / 0022 (Java translates them before lexing, cutting the literal open); + # a three-digit octal escape is exact and cannot swallow a following digit + out.append(f"\\{o:03o}") else: out.append(f"\\u{o:04x}") return '"' + "".join(out) + '"' diff --git a/tests/cobol_mainframe/test_det_bytes_4690.py b/tests/cobol_mainframe/test_det_bytes_4690.py new file mode 100644 index 0000000000..3fad1416ce --- /dev/null +++ b/tests/cobol_mainframe/test_det_bytes_4690.py @@ -0,0 +1,55 @@ +"""#4690 / #4691: every byte value as a COBOL hex literal translates to Java that compiles (a raw or \\u000a-escaped +control character cut the Java string literal open), and DISPLAY writes the record charset's single byte, as the +GnuCOBOL oracle does (the port wrote U+00FF as UTF-8 C3 BF).""" + +from __future__ import annotations + +import os +import shutil + +import pytest +from test_det_programs import ROOT, _cobol, _java, _java_run, program # noqa: F401 + +E2E = pytest.mark.skipif(os.environ.get("EQUIVALENCE_E2E") != "1" or not shutil.which("docker") or _java() is None, + reason="needs Docker and a JDK 17 (JAVA_HOME / JDK_17)") # fmt: skip + + +def every_byte_program(name: str) -> str: + """One MOVE X'hh' TO R and DISPLAY per byte 0x00-0xFF, then a multi-byte literal holding the line breaks.""" + proc: list[str] = [] + for b in range(256): + proc += [f"MOVE X'{b:02X}' TO R", "DISPLAY R"] + proc += ["MOVE X'3132330A0D2230' TO W", "DISPLAY W", "MOVE X'5C5C22' TO W", "DISPLAY W"] + return program(name, ["01 R PIC X.", "01 W PIC X(8)."], proc) + + +@pytest.mark.skipif(_java() is None, reason="needs a JDK 17 (JAVA_HOME / JDK_17)") +def test_every_byte_value_as_a_hex_literal_compiles(tmp_path): + pytest.importorskip("tree_sitter_language_pack") + # _java_run runs javac with check=True: an unclosed string literal raises + _java_run("HEXALL", every_byte_program("HEXALL"), tmp_path, raw=True) + + +@pytest.mark.skipif(_java() is None, reason="needs a JDK 17 (JAVA_HOME / JDK_17)") +def test_jstr_never_emits_a_unicode_escape_for_a_line_break_or_quote(): + from gitgalaxy.tools.cobol_to_java.det.gen import jstr + + for b in range(256): + lit = jstr(chr(b)) + assert "\\u" not in lit, (b, lit) + assert all(ord(c) < 128 for c in lit) + assert jstr("\n\r\x000") == '"\\012\\015\\0000"' + + +@E2E +@pytest.mark.parametrize("mode", ["bytes", "typed", "groups"]) +def test_every_byte_hex_literal_and_display_equal_gnucobols(mode, tmp_path): + pytest.importorskip("tree_sitter_language_pack") + src = every_byte_program("HEXALL") + cob = tmp_path / "cobol" + cob.mkdir() + want = _cobol(src, cob, raw=True) + got = _java_run("HEXALL", src, tmp_path, mode != "bytes", mode == "groups", raw=True) + assert isinstance(want, bytes) and want + assert b"\xc3\xbf" not in got + assert got == want diff --git a/tests/cobol_mainframe/test_det_programs.py b/tests/cobol_mainframe/test_det_programs.py index e5f33ae63f..72981370b0 100644 --- a/tests/cobol_mainframe/test_det_programs.py +++ b/tests/cobol_mainframe/test_det_programs.py @@ -988,17 +988,23 @@ def _java() -> Path | None: return Path(home) / "bin" if home and (Path(home) / "bin/javac").is_file() else None -def _cobol(src: str, work: Path) -> str: +def _cobol(src: str, work: Path, raw: bool = False) -> str: (work / "prog.cbl").write_text(src) run = subprocess.run(["docker", "run", "--rm", "-v", f"{work}:/w", "-w", "/w", IMAGE, "sh", "-c", # noqa: S607 "cobc -x -std=ibm -fsign=EBCDIC prog.cbl -o prog 2>&1 && ./prog"], capture_output=True, - text=True, check=False) # fmt: skip + text=not raw, check=False) # fmt: skip assert run.returncode == 0, run.stdout + run.stderr return run.stdout def _java_run( - name: str, src: str, work: Path, typed: bool = False, groups: bool = False, unit: str | None = None + name: str, + src: str, + work: Path, + typed: bool = False, + groups: bool = False, + unit: str | None = None, + raw: bool = False, ) -> str: from gitgalaxy.tools.cobol_to_java.det import program as P @@ -1026,7 +1032,7 @@ def _java_run( jdk = _java() files = [str(f) for f in srcdir.rglob("*.java")] subprocess.run([str(jdk / "javac"), "-nowarn", "-d", str(work / "classes"), *files], check=True) # noqa: S603 - return subprocess.run([str(jdk / "java"), "-cp", str(work / "classes"), "Main"], capture_output=True, text=True, # noqa: S603 + return subprocess.run([str(jdk / "java"), "-cp", str(work / "classes"), "Main"], capture_output=True, text=not raw, # noqa: S603 check=True).stdout # fmt: skip @@ -1037,6 +1043,7 @@ def _batch_package(srcdir: Path) -> None: for cls in ("CobolFiles", "DatasetResolver", "Dd", "MainframeClock", "Sysout", "CobolAbend"): text = _RUNTIME[cls].replace("{pkg}", f"{PKG}.batch").replace("{zone}", "UTC") + text = text.replace("{record_charset}", "ISO-8859-1") text = re.sub(r"^import org\.springframework\..*\n|^@Component\n", "", text, flags=re.M) text = re.sub(r'@Value\("(?:[^"\\]|\\.)*"\)\s*', "", text) (srcdir / PKG / "batch" / f"{cls}.java").parent.mkdir(parents=True, exist_ok=True) diff --git a/tests/tools/equivalence.py b/tests/tools/equivalence.py index 0998046b1a..0fabd8a7bf 100644 --- a/tests/tools/equivalence.py +++ b/tests/tools/equivalence.py @@ -650,7 +650,10 @@ def sysout_lines(data: bytes, enc: str) -> list[str]: def compare_sysout(cobol: bytes, java: bytes, enc: str) -> dict[str, Any]: """#4056: what each side DISPLAYed, line by line. Not compared (and said so) when the COBOL side DISPLAYed an operand ggdisplay.c does not model: GnuCOBOL's text for it is not IBM's.""" - c, j = sysout_lines(cobol, enc), java.decode("utf-8").split("\n") if java else [] + c, j = ( + sysout_lines(cobol, enc), + java.decode(enc).split("\n") if java else [], + ) # #4691: the port writes record-charset bytes j = [x.rstrip(" \r") for x in j] while j and not j[-1]: j.pop() From 59de3a3565499615699332775be26bca24d4e547 Mon Sep 17 00:00:00 2001 From: squid-protocol Date: Thu, 8 Oct 2026 03:32:11 -0400 Subject: [PATCH 2/2] test: batch Sysout DISPLAY of X'80'..X'FF' vs cobc, latin-1 and non-latin-1 record charsets (#4697) _java_run_batch gains raw= and charset=. Non-latin-1 cases xfail(strict) on #4698. Co-Authored-By: Claude Opus 5.5 --- tests/cobol_mainframe/test_det_bytes_4690.py | 35 +++++++++++++++++++- tests/cobol_mainframe/test_det_programs.py | 21 ++++++++---- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/tests/cobol_mainframe/test_det_bytes_4690.py b/tests/cobol_mainframe/test_det_bytes_4690.py index 3fad1416ce..fd1fb9b3be 100644 --- a/tests/cobol_mainframe/test_det_bytes_4690.py +++ b/tests/cobol_mainframe/test_det_bytes_4690.py @@ -8,7 +8,7 @@ import shutil import pytest -from test_det_programs import ROOT, _cobol, _java, _java_run, program # noqa: F401 +from test_det_programs import ROOT, _cobol, _java, _java_run, _java_run_batch, program # noqa: F401 E2E = pytest.mark.skipif(os.environ.get("EQUIVALENCE_E2E") != "1" or not shutil.which("docker") or _java() is None, reason="needs Docker and a JDK 17 (JAVA_HOME / JDK_17)") # fmt: skip @@ -53,3 +53,36 @@ def test_every_byte_hex_literal_and_display_equal_gnucobols(mode, tmp_path): assert isinstance(want, bytes) and want assert b"\xc3\xbf" not in got assert got == want + + +def high_byte_program(name: str) -> str: + """One MOVE X'hh' TO R and DISPLAY per byte 0x80-0xFF (all above X'7F').""" + proc: list[str] = [] + for b in range(0x80, 0x100): + proc += [f"MOVE X'{b:02X}' TO R", "DISPLAY R"] + return program(name, ["01 R PIC X."], proc) + + +@E2E +@pytest.mark.parametrize( + "charset", + [ + "ISO-8859-1", + *( + pytest.param(c, marks=pytest.mark.xfail(strict=True, reason="#4698: hex-literal bytes do not round-trip")) + for c in ("IBM037", "IBM1047", "windows-1252") + ), + ], +) +@pytest.mark.parametrize("mode", ["bytes", "typed", "groups"]) +def test_batch_sysout_high_bytes_equal_gnucobols(charset, mode, tmp_path): + """#4697: the batch Sysout (not the standalone one) writes the record charset's bytes for DISPLAY of X'80'..X'FF', + byte for byte as cobc does, under latin-1 and under single-byte EBCDIC record charsets.""" + pytest.importorskip("tree_sitter_language_pack") + src = high_byte_program("HIBATCH") + cob = tmp_path / "cobol" + cob.mkdir() + want = _cobol(src, cob, raw=True) + got = _java_run_batch("HIBATCH", src, tmp_path, [], mode != "bytes", mode == "groups", raw=True, charset=charset) + assert isinstance(got, bytes) and want + assert got == want, f"{charset}: java {got[:40]!r} != cobol {want[:40]!r}" diff --git a/tests/cobol_mainframe/test_det_programs.py b/tests/cobol_mainframe/test_det_programs.py index 72981370b0..54f42782e0 100644 --- a/tests/cobol_mainframe/test_det_programs.py +++ b/tests/cobol_mainframe/test_det_programs.py @@ -1036,21 +1036,30 @@ def _java_run( check=True).stdout # fmt: skip -def _batch_package(srcdir: Path) -> None: +def _batch_package(srcdir: Path, charset: str = "ISO-8859-1") -> None: """The generated project's batch package (CobolFiles, DatasetResolver, Dd, MainframeClock, Sysout, CobolAbend) from the batch forge's own templates, without Spring: what a det port with files runs on.""" from gitgalaxy.tools.cobol_to_java.cobol_to_java_batch_forge import _RUNTIME for cls in ("CobolFiles", "DatasetResolver", "Dd", "MainframeClock", "Sysout", "CobolAbend"): text = _RUNTIME[cls].replace("{pkg}", f"{PKG}.batch").replace("{zone}", "UTC") - text = text.replace("{record_charset}", "ISO-8859-1") + text = text.replace("{record_charset}", charset) text = re.sub(r"^import org\.springframework\..*\n|^@Component\n", "", text, flags=re.M) text = re.sub(r'@Value\("(?:[^"\\]|\\.)*"\)\s*', "", text) (srcdir / PKG / "batch" / f"{cls}.java").parent.mkdir(parents=True, exist_ok=True) (srcdir / PKG / "batch" / f"{cls}.java").write_text(text) -def _java_run_batch(name: str, src: str, work: Path, dds: list[str], typed: bool = False, groups: bool = False) -> str: +def _java_run_batch( + name: str, + src: str, + work: Path, + dds: list[str], + typed: bool = False, + groups: bool = False, + raw: bool = False, + charset: str = "ISO-8859-1", +) -> str: """As _java_run, on the batch runtime (DetFiles): runBatch with one DD per dataset, each a file of its name in a datasets directory.""" from gitgalaxy.tools.cobol_to_java.det import program as P @@ -1068,11 +1077,11 @@ def _java_run_batch(name: str, src: str, work: Path, dds: list[str], typed: bool for rel, text in [(f"service/{r.service}.java", java), *runtime.items()]: (srcdir / PKG / rel).parent.mkdir(parents=True, exist_ok=True) (srcdir / PKG / rel).write_text(text) - _batch_package(srcdir) + _batch_package(srcdir, charset) rec = srcdir / PKG / "entity/vsam/CobolRecords.java" rec.parent.mkdir(parents=True, exist_ok=True) rec.write_text(f"package {PKG}.entity.vsam;\npublic final class CobolRecords {{\n public static java.nio.charset." - "Charset charset() {\n return java.nio.charset.StandardCharsets.ISO_8859_1;\n }\n}\n") # fmt: skip + "Charset charset() {\n return java.nio.charset.Charset.forName(\"" + charset + "\");\n }\n}\n") # fmt: skip data = work / "datasets" data.mkdir() b = f"{PKG}.batch" @@ -1085,7 +1094,7 @@ def _java_run_batch(name: str, src: str, work: Path, dds: list[str], typed: bool jdk = _java() files = [str(f) for f in srcdir.rglob("*.java")] subprocess.run([str(jdk / "javac"), "-nowarn", "-d", str(work / "classes"), *files], check=True) # noqa: S603 - return subprocess.run([str(jdk / "java"), "-cp", str(work / "classes"), "Main"], capture_output=True, text=True, # noqa: S603 + return subprocess.run([str(jdk / "java"), "-cp", str(work / "classes"), "Main"], capture_output=True, text=not raw, # noqa: S603 check=True).stdout # fmt: skip