Skip to content

Commit 0cf2055

Browse files
authored
Implement dependency URL rewrites (#154)
As supported by `PklProject` and the Pkl cli. This will parse the http rewrites from `PklProject` and apply them accordingly when downloading packages. See: https://pkl-lang.org/blog/using-packages-in-air-gapped-environments.html#mirror-server
1 parent 091f600 commit 0cf2055

23 files changed

Lines changed: 382 additions & 46 deletions

File tree

.bazelrc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ test --test_output=all
1515
# To update these lines, execute
1616
# `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`.
1717
# This command must be run whenever integration test workspaces are added or removed.
18-
build --deleted_packages=.ijwb/aspect,examples/pets,examples/pkl_project,examples/pkl_project/pkl,examples/simple,tests/integration_tests/example_workspaces/multiple_pkl_projects,tests/integration_tests/example_workspaces/multiple_pkl_projects/project1,tests/integration_tests/example_workspaces/multiple_pkl_projects/project2,tests/integration_tests/example_workspaces/override_version,tests/integration_tests/example_workspaces/pkl_cache,tests/integration_tests/example_workspaces/pkl_package,tests/integration_tests/example_workspaces/pkl_package/pkl_project_basic,tests/integration_tests/example_workspaces/pkl_package/pkl_project_strip_prefix,tests/integration_tests/example_workspaces/pkl_package/pkl_project_version_override,tests/integration_tests/example_workspaces/pkl_package/srcs/pkg2,tests/integration_tests/example_workspaces/pkl_project,tests/integration_tests/example_workspaces/simple
19-
query --deleted_packages=.ijwb/aspect,examples/pets,examples/pkl_project,examples/pkl_project/pkl,examples/simple,tests/integration_tests/example_workspaces/multiple_pkl_projects,tests/integration_tests/example_workspaces/multiple_pkl_projects/project1,tests/integration_tests/example_workspaces/multiple_pkl_projects/project2,tests/integration_tests/example_workspaces/override_version,tests/integration_tests/example_workspaces/pkl_cache,tests/integration_tests/example_workspaces/pkl_package,tests/integration_tests/example_workspaces/pkl_package/pkl_project_basic,tests/integration_tests/example_workspaces/pkl_package/pkl_project_strip_prefix,tests/integration_tests/example_workspaces/pkl_package/pkl_project_version_override,tests/integration_tests/example_workspaces/pkl_package/srcs/pkg2,tests/integration_tests/example_workspaces/pkl_project,tests/integration_tests/example_workspaces/simple
18+
build --deleted_packages=examples/pets,examples/pkl_project,examples/pkl_project/pkl,examples/simple,tests/integration_tests/example_workspaces/mirror_rewrite,tests/integration_tests/example_workspaces/multiple_pkl_projects,tests/integration_tests/example_workspaces/multiple_pkl_projects/project1,tests/integration_tests/example_workspaces/multiple_pkl_projects/project2,tests/integration_tests/example_workspaces/override_version,tests/integration_tests/example_workspaces/pkl_cache,tests/integration_tests/example_workspaces/pkl_package,tests/integration_tests/example_workspaces/pkl_package/pkl_project_basic,tests/integration_tests/example_workspaces/pkl_package/pkl_project_strip_prefix,tests/integration_tests/example_workspaces/pkl_package/pkl_project_version_override,tests/integration_tests/example_workspaces/pkl_package/srcs/pkg2,tests/integration_tests/example_workspaces/pkl_project,tests/integration_tests/example_workspaces/simple
19+
query --deleted_packages=examples/pets,examples/pkl_project,examples/pkl_project/pkl,examples/simple,tests/integration_tests/example_workspaces/mirror_rewrite,tests/integration_tests/example_workspaces/multiple_pkl_projects,tests/integration_tests/example_workspaces/multiple_pkl_projects/project1,tests/integration_tests/example_workspaces/multiple_pkl_projects/project2,tests/integration_tests/example_workspaces/override_version,tests/integration_tests/example_workspaces/pkl_cache,tests/integration_tests/example_workspaces/pkl_package,tests/integration_tests/example_workspaces/pkl_package/pkl_project_basic,tests/integration_tests/example_workspaces/pkl_package/pkl_project_strip_prefix,tests/integration_tests/example_workspaces/pkl_package/pkl_project_version_override,tests/integration_tests/example_workspaces/pkl_package/srcs/pkg2,tests/integration_tests/example_workspaces/pkl_project,tests/integration_tests/example_workspaces/simple
2020

2121
###
2222
# GitHub Actions specific configuration

examples/pkl_project/PklProject

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,14 @@ dependencies {
55
uri = "package://pkg.pkl-lang.org/pkl-k8s/k8s@1.0.1"
66
}
77
}
8+
9+
evaluatorSettings {
10+
http {
11+
rewrites {
12+
// Example of an HTTP rewrite rule.
13+
// This will be picked up by Bazel and applied when downloading dependencies.
14+
// See: https://pkl-lang.org/blog/using-packages-in-air-gapped-environments.html#mirror-server
15+
["https://example.com/"] = "https://my.mirror/example.com/"
16+
}
17+
}
18+
}

pkl/extensions/pkl.bzl

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
1+
# Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@ Module extension for using rules_pkl with bzlmod.
1717
"""
1818

1919
load("//pkl:repositories.bzl", "DEFAULT_PKL_VERSION", "pkl_cli_binaries", "pkl_doc_cli_binaries")
20-
load("//pkl/private:pkl_project.bzl", "parse_pkl_project_deps_json", _pkl_project = "pkl_project")
20+
load("//pkl/private:pkl_project.bzl", "parse_pkl_project_deps_json", "pkl_project_http_rewrites", _pkl_project = "pkl_project")
2121
load("//pkl/private:remote_pkl_package.bzl", "remote_pkl_package")
2222

2323
pkl_project = tag_class(
@@ -81,12 +81,20 @@ def _toolchain_extension(module_ctx):
8181

8282
# Make sure all the remote files are downloaded and unpacked
8383
packages = parse_pkl_project_deps_json(module_ctx.read(proj.pkl_project_deps))
84+
85+
rewrites_repo_name = proj.name + "_http_rewrites"
86+
pkl_project_http_rewrites(
87+
name = rewrites_repo_name,
88+
pkl_project = proj.pkl_project,
89+
)
90+
8491
for package in packages:
8592
if not package.workspace_name in seen_packages:
8693
remote_pkl_package(
8794
name = package.workspace_name,
8895
url = package.url,
8996
sha256 = package.sha256,
97+
rules = "@" + rewrites_repo_name + "//:rules.json",
9098
)
9199
seen_packages.append(package.workspace_name)
92100

pkl/private/pkl_project.bzl

Lines changed: 58 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
1+
# Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -45,6 +45,32 @@ def parse_pkl_project_deps_json(pkl_deps_json_string_path):
4545

4646
return seen_remote_pkl_packages.values()
4747

48+
def _eval_pkl_project(ctx, pkl_project_path, extra_args = []):
49+
"""Evaluates a PklProject file as JSON and returns the decoded object.
50+
51+
Args:
52+
ctx: A repository context or module context.
53+
pkl_project_path: Path to the PklProject file to evaluate.
54+
extra_args: Additional arguments to pass to `pkl eval`.
55+
Returns:
56+
The JSON object output of `pkl eval <pkl_project_path> -f json`.
57+
"""
58+
if ctx.os.name == "linux" and ctx.os.arch == "amd64":
59+
pkl_executable = ctx.path(Label("@pkl-cli-linux-amd64//file:downloaded"))
60+
elif ctx.os.name == "linux" and ctx.os.arch == "aarch64":
61+
pkl_executable = ctx.path(Label("@pkl-cli-linux-aarch64//file:downloaded"))
62+
elif ctx.os.name == "mac os x" and ctx.os.arch == "x86_64":
63+
pkl_executable = ctx.path(Label("@pkl-cli-macos-amd64//file:downloaded"))
64+
elif ctx.os.name == "mac os x" and ctx.os.arch == "aarch64":
65+
pkl_executable = ctx.path(Label("@pkl-cli-macos-aarch64//file:downloaded"))
66+
else:
67+
fail("Couldn't find pkl executable for os {os} and arch {arch}".format(os = ctx.os.name, arch = ctx.os.arch))
68+
69+
result = ctx.execute([pkl_executable, "eval", str(pkl_project_path), "--format", "json"] + extra_args)
70+
if result.return_code != 0:
71+
fail("Error evaluating PklProject: {}".format(result.stderr))
72+
return json.decode(result.stdout)
73+
4874
def convert_dict_to_options(option_name, items_dict):
4975
"""Converts the dictioinary to a list with each "key=value" pair preceded by the option name (e.g. --my-option).
5076
@@ -72,26 +98,8 @@ def _pkl_project_impl(rctx):
7298
rctx.symlink(rctx.attr.pkl_project, "PklProject")
7399
rctx.symlink(rctx.path(rctx.attr.pkl_project_deps), "PklProject.deps.json")
74100

75-
if rctx.os.name == "linux" and rctx.os.arch == "amd64":
76-
pkl_executable = rctx.path(Label("@pkl-cli-linux-amd64//file:downloaded"))
77-
elif rctx.os.name == "linux" and rctx.os.arch == "aarch64":
78-
pkl_executable = rctx.path(Label("@pkl-cli-linux-aarch64//file:downloaded"))
79-
elif rctx.os.name == "mac os x" and rctx.os.arch == "x86_64":
80-
pkl_executable = rctx.path(Label("@pkl-cli-macos-amd64//file:downloaded"))
81-
elif rctx.os.name == "mac os x" and rctx.os.arch == "aarch64":
82-
pkl_executable = rctx.path(Label("@pkl-cli-macos-aarch64//file:downloaded"))
83-
else:
84-
fail("Couldn't find pkl executable for os {os} and arch {arch}".format(os = rctx.os.name, arch = rctx.os.arch))
85-
86101
env_vars = convert_dict_to_options("--env-var", rctx.attr.environment)
87-
rendered_result = rctx.execute(
88-
["{}".format(pkl_executable), "eval", "PklProject", "-f", "json"] + env_vars + rctx.attr.extra_flags,
89-
)
90-
if rendered_result.return_code != 0:
91-
fail("Error evaluating and rendering PklProject file as json: {}".format(rendered_result.stderr))
92-
metadata = rendered_result.stdout
93-
94-
pkl_project_metadata = json.decode(metadata)
102+
pkl_project_metadata = _eval_pkl_project(rctx, "PklProject", extra_args = env_vars + rctx.attr.extra_flags)
95103
has_package = "package" in pkl_project_metadata
96104

97105
build_bazel_content = ""
@@ -129,6 +137,36 @@ pkl_cache(
129137

130138
rctx.file("BUILD.bazel", content = build_bazel_content, executable = False)
131139

140+
def _pkl_project_http_rewrites_impl(rctx):
141+
rules = _eval_pkl_project(rctx, rctx.path(rctx.attr.pkl_project), extra_args = [
142+
"--expression",
143+
# Render the rewrite rules as list, to make sure the order is preserved.
144+
"""
145+
new JsonRenderer {}
146+
.renderValue(
147+
(evaluatorSettings?.http?.rewrites?.toMap() ?? Map())
148+
.entries
149+
.map(
150+
(_rule) -> Map(_rule.key, _rule.value)
151+
)
152+
)
153+
""",
154+
])
155+
rctx.file("rules.json", content = json.encode(rules))
156+
rctx.file("BUILD.bazel", content = 'exports_files(["rules.json"])\n')
157+
158+
pkl_project_http_rewrites = repository_rule(
159+
_pkl_project_http_rewrites_impl,
160+
doc = "Evaluates a PklProject file once to extract HTTP rewrite rules.",
161+
attrs = {
162+
"pkl_project": attr.label(
163+
doc = "The PklProject file to evaluate for HTTP rewrite rules.",
164+
allow_single_file = True,
165+
mandatory = True,
166+
),
167+
},
168+
)
169+
132170
pkl_project = repository_rule(
133171
_pkl_project_impl,
134172
doc = """

pkl/private/remote_pkl_package.bzl

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
1+
# Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -18,6 +18,21 @@ Repository rule for downloading remote Pkl packages.
1818

1919
load(":pkl_package_names.bzl", "get_terminal_package_name")
2020

21+
def rewrite_url(url, rules):
22+
"""Returns the rewritten URL if a mirror prefix matches, else the original URL.
23+
24+
Args:
25+
url: The original URL to potentially rewrite.
26+
rules: A list mapping original URL prefixes to mirror URL prefixes.
27+
Returns:
28+
The rewritten URL if any of the rules matched. Otherwise the original URL.
29+
"""
30+
for rule in rules:
31+
original, mirror = rule.items()[0]
32+
if url.startswith(original):
33+
return mirror + url[len(original):]
34+
return url
35+
2136
def _remote_pkl_package_impl(rctx):
2237
if not rctx.attr.url.startswith("projectpackage://"):
2338
fail("URL does not look like a Pkl project:", rctx.attr.url)
@@ -30,11 +45,17 @@ def _remote_pkl_package_impl(rctx):
3045
metadata_file = "package-2/%s/%s.json" % (url_without_scheme, file_name)
3146
package_archive = "package-2/%s/%s.zip" % (url_without_scheme, file_name)
3247

33-
# Grab the JSON from the original location
34-
rctx.download(url, sha256 = rctx.attr.sha256, output = metadata_file)
48+
rules = []
49+
if rctx.attr.rules:
50+
rules = json.decode(rctx.read(rctx.path(rctx.attr.rules)))
51+
52+
# Grab the JSON from the original location, apply any rewrite rules.
53+
rctx.download([rewrite_url(url, rules)], sha256 = rctx.attr.sha256, output = metadata_file)
3554

3655
metadata = json.decode(rctx.read(metadata_file))
37-
rctx.download(metadata["packageZipUrl"], sha256 = metadata["packageZipChecksums"]["sha256"], output = package_archive)
56+
57+
# Download the package ZIP, apply any rewrite rules.
58+
rctx.download([rewrite_url(metadata["packageZipUrl"], rules)], sha256 = metadata["packageZipChecksums"]["sha256"], output = package_archive)
3859

3960
rctx.file(
4061
"BUILD.bazel",
@@ -63,5 +84,10 @@ remote_pkl_package = repository_rule(
6384
doc = "SHA256 hash of the package's metadata file.",
6485
mandatory = True,
6586
),
87+
"rules": attr.label(
88+
doc = """A JSON file containing a list of URL prefix rewrite rules.
89+
Typically provided by the pkl_project_http_rewrites repository rule.""",
90+
allow_single_file = True,
91+
),
6692
},
6793
)

tests/BUILD.bazel

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
1+
# Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -12,10 +12,15 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
load(":remote_pkl_package_test.bzl", "rewrite_url_test_suite")
1516
load(":repositories_test.bzl", "project_cache_path_and_dependencies_test_suite")
1617

1718
package(default_visibility = ["//visibility:private"])
1819

1920
project_cache_path_and_dependencies_test_suite(
2021
name = "project_cache_path_and_dependencies_tests",
2122
)
23+
24+
rewrite_url_test_suite(
25+
name = "rewrite_url_tests",
26+
)

tests/integration_tests/BUILD.bazel

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
1+
# Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -114,6 +114,21 @@ bazel_integration_tests(
114114
workspace_path = "example_workspaces/pkl_project",
115115
)
116116

117+
bazel_integration_tests(
118+
name = "mirror_rewrite_test",
119+
bazel_versions = bazel_binaries.versions.all,
120+
tags = integration_test_utils.DEFAULT_INTEGRATION_TEST_TAGS + [
121+
# Add 'no-sandbox' tag to disable running this test target in the Bazel sandbox to prevent
122+
# file permission issues when accessing/writing to the cache.
123+
"no-sandbox",
124+
],
125+
test_runner = ":simple_test_runner",
126+
workspace_files = integration_test_utils.glob_workspace_files("example_workspaces/mirror_rewrite") + [
127+
"//:workspace_files",
128+
],
129+
workspace_path = "example_workspaces/mirror_rewrite",
130+
)
131+
117132
# By default, the integration test targets are tagged as `manual`, so we need
118133
# define a `test_suite` target for the integration test to be executed during
119134
# bazel test //...
@@ -143,6 +158,10 @@ test_suite(
143158
integration_test_utils.bazel_integration_test_names(
144159
":pkl_project_test",
145160
bazel_binaries.versions.all,
161+
) +
162+
integration_test_utils.bazel_integration_test_names(
163+
":mirror_rewrite_test",
164+
bazel_binaries.versions.all,
146165
),
147166
visibility = ["//:__subpackages__"],
148167
)
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
common --enable_bzlmod
2+
3+
test --test_output=errors
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
7.6.1
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
load("@aspect_bazel_lib//lib:diff_test.bzl", "diff_test")
16+
load("@rules_pkl//pkl:defs.bzl", "pkl_eval")
17+
18+
pkl_eval(
19+
name = "pkl_eval_with_mirror",
20+
srcs = [
21+
"example.pkl",
22+
],
23+
entrypoints = [
24+
"example.pkl",
25+
],
26+
format = "yaml",
27+
deps = [
28+
"@mypkl//:packages",
29+
],
30+
)
31+
32+
diff_test(
33+
name = "pkl_eval_with_mirror_diff_test",
34+
file1 = "expected_output.yaml",
35+
file2 = ":pkl_eval_with_mirror",
36+
)

0 commit comments

Comments
 (0)