Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/config-schema.md
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,7 @@ This document describes the schema for the librarian.yaml.

| Field | Type | Description |
| :--- | :--- | :--- |
| `alternate_headers` | string | Is the path to a file containing alternate license header text. |
| `api_id_override` | string | Is the ID of the API (e.g., "pubsub.googleapis.com"), allows the "api_id" field in .repo-metadata.json to be overridden. Defaults to "{library.api_shortname}.googleapis.com". |
| `api_reference` | string | Is the URL for the API reference documentation. |
| `api_description_override` | string | Allows the "api_description" field in .repo-metadata.json to be overridden. |
Expand Down
3 changes: 3 additions & 0 deletions internal/config/language.go
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,9 @@ type DartPackage struct {
// TODO(https://github.com/googleapis/librarian/issues/4130):
// add fill defaults for fields with default.
type JavaModule struct {
// AlternateHeaders is the path to a file containing alternate license header text.
AlternateHeaders string `yaml:"alternate_headers,omitempty"`

// APIIDOverride is the ID of the API (e.g., "pubsub.googleapis.com"),
// allows the "api_id" field in .repo-metadata.json to be overridden.
// Defaults to "{library.api_shortname}.googleapis.com".
Expand Down
142 changes: 85 additions & 57 deletions internal/librarian/java/postprocess.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,91 +67,97 @@ type libraryPostProcessParams struct {
transports map[string]serviceconfig.Transport
}

func postProcessLibrary(ctx context.Context, p libraryPostProcessParams) error {
if err := createOrVerifyOwlbotPy(p.outDir); err != nil {
func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) error {
if err := createOrVerifyOwlbotPy(params.outDir); err != nil {
return err
}
bomVersion, err := findBOMVersion(p.cfg)
bomVersion, err := findBOMVersion(params.cfg)
if err != nil {
return err
}
if err := removeKeptFilesFromStaging(p.library, p.outDir); err != nil {
if err := removeKeptFilesFromStaging(params.library, params.outDir); err != nil {
return fmt.Errorf("failed to remove kept files from staging: %w", err)
}
if err := runOwlBot(ctx, p.library, p.outDir, bomVersion); err != nil {
if err := runOwlBot(ctx, params.library, params.outDir, bomVersion); err != nil {
return fmt.Errorf("%w: %w", errRunOwlBot, err)
}

monorepoVersion, err := findMonorepoVersion(p.cfg)
monorepoVersion, err := findMonorepoVersion(params.cfg)
if err != nil {
return err
}
if err := syncPOMs(p.library, p.outDir, monorepoVersion, p.metadata, p.transports); err != nil {
if err := syncPOMs(params.library, params.outDir, monorepoVersion, params.metadata, params.transports); err != nil {
return fmt.Errorf("%w: %w", errSyncPOMs, err)
}

return nil
}

func (p postProcessParams) gapicDir() string { return filepath.Join(p.outDir, p.apiBase, "gapic") }
func (p postProcessParams) gRPCDir() string { return filepath.Join(p.outDir, p.apiBase, "grpc") }
func (p postProcessParams) protoDir() string { return filepath.Join(p.outDir, p.apiBase, "proto") }
func (p postProcessParams) coords() APICoordinate {
return DeriveAPICoordinates(DeriveLibraryCoordinates(p.library), p.apiBase, p.javaAPI)
func (params postProcessParams) gapicDir() string {
return filepath.Join(params.outDir, params.apiBase, "gapic")
}
func (params postProcessParams) gRPCDir() string {
return filepath.Join(params.outDir, params.apiBase, "grpc")
}
func (params postProcessParams) protoDir() string {
return filepath.Join(params.outDir, params.apiBase, "proto")
}
func (params postProcessParams) coords() APICoordinate {
return DeriveAPICoordinates(DeriveLibraryCoordinates(params.library), params.apiBase, params.javaAPI)
}

func stagingDir(outDir string) string { return filepath.Join(outDir, owlbotStagingDir) }

func postProcessAPI(ctx context.Context, p postProcessParams) error {
gapicDir := p.gapicDir()
gRPCDir := p.gRPCDir()
protoDir := p.protoDir()
func postProcessAPI(ctx context.Context, params postProcessParams) error {
gapicDir := params.gapicDir()
gRPCDir := params.gRPCDir()
protoDir := params.protoDir()
// Unzip the temp-codegen.srcjar into temporary {gapicDir} directory.
srcjarPath := filepath.Join(gapicDir, "temp-codegen.srcjar")
if _, err := os.Stat(srcjarPath); err == nil {
if err := filesystem.Unzip(ctx, srcjarPath, gapicDir); err != nil {
return fmt.Errorf("failed to unzip %s: %w", srcjarPath, err)
}
}
if err := addHeadersIfRequired(p, []string{gRPCDir, protoDir}); err != nil {
if err := addHeaders(params, []string{gRPCDir, protoDir}); err != nil {
return err
}
if err := copyFiles(p); err != nil {
if err := copyFiles(params); err != nil {
return fmt.Errorf("failed to copy files: %w", err)
}
if err := restructureToStaging(p); err != nil {
if err := restructureToStaging(params); err != nil {
return fmt.Errorf("failed to restructure to staging: %w", err)
}

// Generate clirr-ignored-differences.xml for the proto module.
// We target the staging directory because runOwlBot hasn't moved the files
// to their final destination yet.
coords := p.coords()
protoModuleRepoRoot := filepath.Join(p.outDir, coords.Proto.ArtifactID)
shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, p.javaAPI.Monolithic)
coords := params.coords()
protoModuleRepoRoot := filepath.Join(params.outDir, coords.Proto.ArtifactID)
shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, params.javaAPI.Monolithic)
if err != nil {
return fmt.Errorf("failed to check for clirr ignore file: %w", err)
}
if shouldGenerate {
protoModuleStagingRoot := filepath.Join(stagingDir(p.outDir), p.apiBase, coords.Proto.ArtifactID)
protoModuleStagingRoot := filepath.Join(stagingDir(params.outDir), params.apiBase, coords.Proto.ArtifactID)
if err := generateClirrIgnore(protoModuleStagingRoot); err != nil {
return fmt.Errorf("failed to generate clirr ignore file: %w", err)
}
}

// Cleanup intermediate protoc output directory after restructuring
if err := os.RemoveAll(filepath.Join(p.outDir, p.apiBase)); err != nil {
if err := os.RemoveAll(filepath.Join(params.outDir, params.apiBase)); err != nil {
return fmt.Errorf("failed to cleanup intermediate files: %w", err)
}
return nil
}

func addHeadersIfRequired(p postProcessParams, dirs []string) error {
if p.javaAPI.Monolithic {
func addHeaders(params postProcessParams, dirs []string) error {
if params.javaAPI.Monolithic {
return nil
}
for _, dir := range dirs {
if err := addMissingHeaders(dir); err != nil {
if err := addMissingHeaders(params, dir); err != nil {
return fmt.Errorf("failed to fix headers in %s: %w", dir, err)
}
}
Expand All @@ -160,9 +166,11 @@ func addHeadersIfRequired(p postProcessParams, dirs []string) error {

// addMissingHeaders prepends the license header to all Java files in the given directory
// if they don't already have one.
func addMissingHeaders(dir string) error {
year := time.Now().Year()
licenseText := buildLicenseText(year)
func addMissingHeaders(params postProcessParams, dir string) error {
headerText, err := getLicenseText(params)
if err != nil {
return err
}
return filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
if err != nil || !d.Type().IsRegular() || filepath.Ext(path) != ".java" {
return err
Expand All @@ -174,16 +182,36 @@ func addMissingHeaders(dir string) error {
if license.HasHeader(content) {
return nil
}
return os.WriteFile(path, append([]byte(licenseText), content...), 0644)
return os.WriteFile(path, append(headerText, content...), 0644)
})
}

func copyFiles(p postProcessParams) error {
if p.javaAPI == nil || len(p.javaAPI.CopyFiles) == 0 {
// getLicenseText reads the contents of the alternate_header property (a filepath)
// if a library has an alternate header file. Otherwise it will grab the default license
// header.
func getLicenseText(params postProcessParams) ([]byte, error) {
if params.library == nil || params.library.Java == nil || params.library.Java.AlternateHeaders == "" {
year := time.Now().Year()
return []byte(buildLicenseText(year)), nil
}
headerPath := filepath.Join(params.outDir, params.library.Java.AlternateHeaders)
b, err := os.ReadFile(headerPath)
if err != nil {
return nil, fmt.Errorf("failed to read alternate header file %s: %w", headerPath, err)
}
// Ensure the alternate header ends with a newline before it is prepended.
if len(b) > 0 && b[len(b)-1] != '\n' {
b = append(b, '\n')
}
return b, nil
}

func copyFiles(params postProcessParams) error {
if params.javaAPI == nil || len(params.javaAPI.CopyFiles) == 0 {
return nil
}
gapicDir := p.gapicDir()
for _, c := range p.javaAPI.CopyFiles {
gapicDir := params.gapicDir()
for _, c := range params.javaAPI.CopyFiles {
src := filepath.Join(gapicDir, c.Source)
dest := filepath.Join(gapicDir, c.Destination)
if _, err := os.Stat(src); err != nil {
Expand Down Expand Up @@ -230,16 +258,16 @@ func removeConflictingFiles(protoSrcDir string) error {
// that matches the structure expected by owlbot.py. It nests modules under the
// {apiBase} directory (e.g., owl-bot-staging/v1/proto-google-cloud-chat-v1) to
// ensure synthtool preserves the module structure.
func restructureToStaging(p postProcessParams) error {
stagingDir := stagingDir(p.outDir)
destRoot := filepath.Join(stagingDir, p.apiBase)
if p.javaAPI.Monolithic {
func restructureToStaging(params postProcessParams) error {
stagingDir := stagingDir(params.outDir)
destRoot := filepath.Join(stagingDir, params.apiBase)
if params.javaAPI.Monolithic {
destRoot = filepath.Join(destRoot, "src")
}
if err := os.MkdirAll(destRoot, 0755); err != nil {
return fmt.Errorf("failed to create staging directory: %w", err)
}
return restructureModules(p, destRoot)
return restructureModules(params, destRoot)
}

type moveAction struct {
Expand All @@ -264,10 +292,10 @@ func restructure(actions []moveAction) error {
// restructureModules moves the generated code from the temporary versioned directory
// tree into the destination root directory for GAPIC, Proto, gRPC, and samples.
// It also copies the relevant proto files into the proto module.
func restructureModules(p postProcessParams, destRoot string) error {
coords := p.coords()
tempProtoSrcDir := p.protoDir()
if p.library.Name != commonProtosLibrary {
func restructureModules(params postProcessParams, destRoot string) error {
coords := params.coords()
tempProtoSrcDir := params.protoDir()
if params.library.Name != commonProtosLibrary {
if err := removeConflictingFiles(tempProtoSrcDir); err != nil {
return err
}
Expand All @@ -279,7 +307,7 @@ func restructureModules(p postProcessParams, destRoot string) error {
gapicTestDest := filepath.Join(destRoot, coords.GAPIC.ArtifactID, "src", "test")
protoFilesDestDir := filepath.Join(destRoot, coords.Proto.ArtifactID, "src", "main", "proto")

if p.javaAPI.Monolithic {
if params.javaAPI.Monolithic {
protoDest = filepath.Join(destRoot, "main", "java")
grpcDest = filepath.Join(destRoot, "main", "java")
gapicMainDest = filepath.Join(destRoot, "main")
Expand All @@ -288,44 +316,44 @@ func restructureModules(p postProcessParams, destRoot string) error {
}

var actions []moveAction
if shouldGenerateProto(p.javaAPI) {
if shouldGenerateProto(params.javaAPI) {
actions = append(actions, moveAction{
src: tempProtoSrcDir,
dest: protoDest,
description: "proto source",
})
}
if shouldGenerateGRPC(p.javaAPI) {
if shouldGenerateGRPC(params.javaAPI) {
actions = append(actions, moveAction{
src: p.gRPCDir(),
src: params.gRPCDir(),
dest: grpcDest,
description: "grpc source",
})
}
if shouldGenerateGAPIC(p.javaAPI) {
if shouldGenerateGAPIC(params.javaAPI) {
actions = append(actions, []moveAction{
{
src: filepath.Join(p.gapicDir(), "src", "main"),
src: filepath.Join(params.gapicDir(), "src", "main"),
dest: gapicMainDest,
description: "gapic source",
},
{
src: filepath.Join(p.gapicDir(), "src", "test"),
src: filepath.Join(params.gapicDir(), "src", "test"),
dest: gapicTestDest,
description: "gapic test",
},
}...)
}
if shouldGenerateResourceNames(p.javaAPI) {
if shouldGenerateResourceNames(params.javaAPI) {
actions = append(actions, moveAction{
src: filepath.Join(p.gapicDir(), "proto", "src", "main", "java"),
src: filepath.Join(params.gapicDir(), "proto", "src", "main", "java"),
dest: protoDest,
description: "resource name source",
})
}
if p.includeSamples && shouldGenerateGAPIC(p.javaAPI) {
if params.includeSamples && shouldGenerateGAPIC(params.javaAPI) {
actions = append(actions, moveAction{
src: filepath.Join(p.gapicDir(), "samples", "snippets", "generated", "src", "main", "java"),
src: filepath.Join(params.gapicDir(), "samples", "snippets", "generated", "src", "main", "java"),
dest: filepath.Join(destRoot, "samples", "snippets", "generated"),
description: "samples",
})
Expand All @@ -334,8 +362,8 @@ func restructureModules(p postProcessParams, destRoot string) error {
return err
}
// Copy proto files to proto-*/src/main/proto
if shouldGenerateProto(p.javaAPI) {
if err := copyProtos(p.protosToCopy, protoFilesDestDir); err != nil {
if shouldGenerateProto(params.javaAPI) {
if err := copyProtos(params.protosToCopy, protoFilesDestDir); err != nil {
return fmt.Errorf("failed to copy proto files: %w", err)
}
}
Expand Down
Loading
Loading