diff --git a/.github/workflows/on-push-branch.yml b/.github/workflows/on-push-branch.yml index cece856b..c7069364 100644 --- a/.github/workflows/on-push-branch.yml +++ b/.github/workflows/on-push-branch.yml @@ -35,7 +35,7 @@ jobs: - name: Build Self run: | echo "# Self Build" >> $GITHUB_STEP_SUMMARY - make self config=Release terrabuild=$PWD/.out/dotnet/terrabuild + make self config=Release - name: Self Integration Tests run: make smoke-tests terrabuild=$PWD/.out/dotnet/terrabuild diff --git a/Makefile b/Makefile index 173dcf48..1a2cecef 100644 --- a/Makefile +++ b/Makefile @@ -73,10 +73,10 @@ docs: dotnet run --project tools/DocGen -- src/Terrabuild.Extensions/bin/$(config)/net9.0/Terrabuild.Extensions.xml ../terrabuild.io/content/docs/extensions self: clean publish - $(terrabuild) run build test dist --configuration $(config) --retry --debug --log --local-only + $(PWD)/.out/dotnet/terrabuild run build test dist --configuration $(config) --debug --log --retry --local-only self-logs: - $(terrabuild) logs build test dist --configuration $(config) --debug --log --local-only + $(PWD)/.out/dotnet/terrabuild logs build test dist --configuration $(config) --debug --log --local-only logs: $(terrabuild) logs build test dist --configuration $(config) --debug --log --local-only diff --git a/src/Terrabuild.Configuration.Tests/Project.fs b/src/Terrabuild.Configuration.Tests/Project.fs index eae0988d..29cdeebe 100644 --- a/src/Terrabuild.Configuration.Tests/Project.fs +++ b/src/Terrabuild.Configuration.Tests/Project.fs @@ -47,12 +47,14 @@ let parseProject() = TargetBlock.Rebuild = None TargetBlock.Outputs = None TargetBlock.Cache = None + TargetBlock.Managed = None TargetBlock.Steps = [ { Extension = "@dotnet"; Command = "build"; Parameters = Map.empty } ] } let targetDist = { TargetBlock.DependsOn = None TargetBlock.Rebuild = None TargetBlock.Outputs = None TargetBlock.Cache = None + TargetBlock.Managed = Expr.False |> Some TargetBlock.Steps = [ { Extension = "@dotnet"; Command = "build"; Parameters = Map.empty } { Extension = "@dotnet"; Command = "publish"; Parameters = Map.empty } ] } let targetDocker = @@ -60,6 +62,7 @@ let parseProject() = TargetBlock.Rebuild = Some (Expr.Bool false) TargetBlock.Outputs = None TargetBlock.Cache = "always" |> Expr.String |> Some + TargetBlock.Managed = None TargetBlock.Steps = [ { Extension = "@shell"; Command = "echo" Parameters = Map [ "arguments", Expr.Function (Function.Trim, [ Expr.Function (Function.Plus, @@ -115,6 +118,7 @@ let parseProject2() = Expr.String ".dll" ])] |> Some TargetBlock.DependsOn = None TargetBlock.Cache = None + TargetBlock.Managed = None TargetBlock.Steps = [ { Extension = "@dotnet"; Command = "build"; Parameters = Map.empty } ] } let locals = diff --git a/src/Terrabuild.Configuration.Tests/TestFiles/Success_PROJECT b/src/Terrabuild.Configuration.Tests/TestFiles/Success_PROJECT index 0c9b1b23..bf922a25 100644 --- a/src/Terrabuild.Configuration.Tests/TestFiles/Success_PROJECT +++ b/src/Terrabuild.Configuration.Tests/TestFiles/Success_PROJECT @@ -37,6 +37,7 @@ target build { } target dist { + managed = false @dotnet build { } @dotnet publish { } } diff --git a/src/Terrabuild.Configuration/AST/Project.fs b/src/Terrabuild.Configuration/AST/Project.fs index 0180bef9..21cd499a 100644 --- a/src/Terrabuild.Configuration/AST/Project.fs +++ b/src/Terrabuild.Configuration/AST/Project.fs @@ -26,6 +26,7 @@ type TargetBlock = Outputs: Expr option DependsOn: Set option Cache: Expr option + Managed: Expr option Steps: Step list } [] diff --git a/src/Terrabuild.Configuration/Transpiler/Project.fs b/src/Terrabuild.Configuration/Transpiler/Project.fs index 41631d37..ee90f2e8 100644 --- a/src/Terrabuild.Configuration/Transpiler/Project.fs +++ b/src/Terrabuild.Configuration/Transpiler/Project.fs @@ -71,7 +71,7 @@ let toProject (block: Block) = let toTarget (block: Block) = block - |> checkAllowedAttributes ["rebuild"; "outputs"; "depends_on"; "cache"] + |> checkAllowedAttributes ["rebuild"; "outputs"; "depends_on"; "cache"; "managed"] |> ignore let rebuild = block |> tryFindAttribute "rebuild" @@ -85,6 +85,7 @@ let toTarget (block: Block) = | String.Regex "^target\.(.*)$" [targetIdentifier] -> targetIdentifier | _ -> raiseInvalidArg $"Invalid target dependency '{dependency}'")) let cache = block |> tryFindAttribute "cache" + let managed = block |> tryFindAttribute "managed" let steps = block.Blocks |> List.map (fun step -> @@ -110,7 +111,8 @@ let toTarget (block: Block) = TargetBlock.Outputs = outputs TargetBlock.DependsOn = dependsOn TargetBlock.Cache = cache - TargetBlock.Steps = steps } + TargetBlock.Steps = steps + TargetBlock.Managed = managed } diff --git a/src/Terrabuild.Extensibility/Extensions.fs b/src/Terrabuild.Extensibility/Extensions.fs index bc8fc56f..1253c5bb 100644 --- a/src/Terrabuild.Extensibility/Extensions.fs +++ b/src/Terrabuild.Extensibility/Extensions.fs @@ -50,13 +50,15 @@ type ShellOperations = ShellOperation list type ActionExecutionRequest = { Cache: Cacheability Operations: ShellOperations + SideEffect: bool } -let shellOp cmd args = +let shellOp(cmd, args) = { ShellOperation.Command = cmd ShellOperation.Arguments = args } -let execRequest cache ops = +let execRequest(cache, ops, sideEffect) = { ActionExecutionRequest.Cache = cache - ActionExecutionRequest.Operations = ops } + ActionExecutionRequest.Operations = ops + ActionExecutionRequest.SideEffect = sideEffect } diff --git a/src/Terrabuild.Extensions/Cargo.fs b/src/Terrabuild.Extensions/Cargo.fs index d04068bf..1dab93be 100644 --- a/src/Terrabuild.Extensions/Cargo.fs +++ b/src/Terrabuild.Extensions/Cargo.fs @@ -47,8 +47,8 @@ type Cargo() = let arguments = arguments |> Option.defaultValue "" let arguments = $"{context.Command} {arguments}" - let ops = [ shellOp "cargo" arguments ] - execRequest Cacheability.Always ops + let ops = [ shellOp("cargo", arguments) ] + execRequest(Cacheability.Always, ops, false) /// @@ -60,8 +60,8 @@ type Cargo() = let profile = profile |> Option.defaultValue "dev" let arguments = arguments |> Option.defaultValue "" - let ops = [ shellOp "cargo" $"build --profile {profile} {arguments}" ] - execRequest Cacheability.Always ops + let ops = [ shellOp("cargo", $"build --profile {profile} {arguments}") ] + execRequest(Cacheability.Always, ops, false) /// @@ -73,5 +73,5 @@ type Cargo() = let profile = profile |> Option.defaultValue "dev" let arguments = arguments |> Option.defaultValue "" - let ops = [ shellOp "cargo" $"test --profile {profile} {arguments}" ] - execRequest Cacheability.Always ops + let ops = [ shellOp("cargo", $"test --profile {profile} {arguments}") ] + execRequest(Cacheability.Always, ops, false) diff --git a/src/Terrabuild.Extensions/Docker.fs b/src/Terrabuild.Extensions/Docker.fs index e6ec7ed3..a76b4381 100644 --- a/src/Terrabuild.Extensions/Docker.fs +++ b/src/Terrabuild.Extensions/Docker.fs @@ -16,8 +16,8 @@ type Docker() = let arguments = arguments |> Option.defaultValue "" let arguments = $"{context.Command} {arguments}" - let ops = [ shellOp "docker" arguments ] - execRequest Cacheability.Always ops + let ops = [ shellOp("docker", arguments) ] + execRequest(Cacheability.Always, ops, true) /// @@ -42,15 +42,15 @@ type Docker() = let ops = [ let buildArgs = $"build --file {dockerfile} --tag {image}:{context.Hash}{args}{platformArgs} ." - shellOp "docker" buildArgs - if context.CI then shellOp "docker" $"push {image}:{context.Hash}" + shellOp("docker", buildArgs) + if context.CI then shellOp("docker", $"push {image}:{context.Hash}") ] let cacheability = if context.CI then Cacheability.Remote else Cacheability.Local - execRequest cacheability ops + execRequest(cacheability, ops, true) /// @@ -62,13 +62,13 @@ type Docker() = let ops = [ if context.CI then - shellOp "docker" $"buildx imagetools create -t {image}:{tag} {image}:{context.Hash}" + shellOp("docker", $"buildx imagetools create -t {image}:{tag} {image}:{context.Hash}") else - shellOp "docker" $"tag {image}:{context.Hash} {image}:{tag}" + shellOp("docker", $"tag {image}:{context.Hash} {image}:{tag}") ] let cacheability = if context.CI then Cacheability.Remote else Cacheability.Local - execRequest cacheability ops + execRequest(cacheability, ops, true) diff --git a/src/Terrabuild.Extensions/Dotnet.fs b/src/Terrabuild.Extensions/Dotnet.fs index 13287821..87329587 100644 --- a/src/Terrabuild.Extensions/Dotnet.fs +++ b/src/Terrabuild.Extensions/Dotnet.fs @@ -56,9 +56,6 @@ module DotnetHelpers = /// type Dotnet() = - static let buildRequest (context: ActionContext) buildOps = - execRequest Cacheability.Always buildOps - /// /// Provides default values for project. /// @@ -85,8 +82,8 @@ type Dotnet() = let arguments = arguments |> Option.defaultValue "" let arguments = $"{context.Command} {arguments}" - let ops = [ shellOp "dotnet" arguments ] - execRequest Cacheability.Always ops + let ops = [ shellOp("dotnet", arguments) ] + execRequest(Cacheability.Always, ops, false) /// @@ -118,11 +115,11 @@ type Dotnet() = let arguments = arguments |> Option.defaultValue "" - let buildOps = [ - shellOp "dotnet" $"build --no-dependencies --configuration {configuration} {logger} {maxcpucount} {version} {arguments}" + let ops = [ + shellOp("dotnet", $"build --no-dependencies --configuration {configuration} {logger} {maxcpucount} {version} {arguments}") ] - buildRequest context buildOps + execRequest(Cacheability.Always, ops, false) /// @@ -136,11 +133,11 @@ type Dotnet() = let version = version |> Option.defaultValue "0.0.0" let arguments = arguments |> Option.defaultValue "" - let buildOps = [ - shellOp "dotnet" $"pack --no-build --configuration {configuration} /p:Version={version} /p:TargetsForTfmSpecificContentInPackage= {arguments}" + let ops = [ + shellOp("dotnet", $"pack --no-build --configuration {configuration} /p:Version={version} /p:TargetsForTfmSpecificContentInPackage= {arguments}") ] - buildRequest context buildOps + execRequest(Cacheability.Always, ops, false) /// /// Publish a project. @@ -167,11 +164,11 @@ type Dotnet() = | _ -> "" let arguments = arguments |> Option.defaultValue "" - let buildOps = [ - shellOp "dotnet" $"publish --no-dependencies --configuration {configuration} {runtime} {trim} {single} {arguments}" + let ops = [ + shellOp("dotnet", $"publish --no-dependencies --configuration {configuration} {runtime} {trim} {single} {arguments}") ] - buildRequest context buildOps + execRequest(Cacheability.Always, ops, false) /// /// Restore packages. @@ -181,8 +178,8 @@ type Dotnet() = static member restore (arguments: string option) = let arguments = arguments |> Option.defaultValue "" - let ops = [ shellOp "dotnet" $"restore {arguments}" ] - execRequest Cacheability.Local ops + let ops = [ shellOp( "dotnet", $"restore {arguments}") ] + execRequest(Cacheability.Local, ops, false) /// @@ -196,8 +193,8 @@ type Dotnet() = let filter = filter |> Option.map (fun filter -> $" --filter \"{filter}\"") |> Option.defaultValue "" let arguments = arguments |> Option.defaultValue "" - let buildOps = [ - shellOp "dotnet" $"test --no-build --configuration {configuration} {filter} {arguments}" + let ops = [ + shellOp("dotnet", $"test --no-build --configuration {configuration} {filter} {arguments}") ] - buildRequest context buildOps + execRequest(Cacheability.Local, ops, false) diff --git a/src/Terrabuild.Extensions/Gradle.fs b/src/Terrabuild.Extensions/Gradle.fs index 70131dbf..c3dafa83 100644 --- a/src/Terrabuild.Extensions/Gradle.fs +++ b/src/Terrabuild.Extensions/Gradle.fs @@ -30,8 +30,8 @@ type Gradle() = let arguments = arguments |> Option.defaultValue "" let arguments = $"{context.Command} {arguments}" - let ops = [ shellOp "gradle" arguments ] - execRequest Cacheability.Always ops + let ops = [ shellOp("gradle", arguments) ] + execRequest(Cacheability.Always, ops, false) /// @@ -41,6 +41,5 @@ type Gradle() = static member build (context: ActionContext) (configuration: string option) = let configuration = configuration |> Option.defaultValue GradleHelpers.defaultConfiguration - let ops = [ shellOp "gradlew" $"assemble{configuration}" ] - - execRequest Cacheability.Always ops + let ops = [ shellOp("gradlew", $"assemble{configuration}") ] + execRequest(Cacheability.Always, ops, false) diff --git a/src/Terrabuild.Extensions/Make.fs b/src/Terrabuild.Extensions/Make.fs index 79156991..ba2b2da9 100644 --- a/src/Terrabuild.Extensions/Make.fs +++ b/src/Terrabuild.Extensions/Make.fs @@ -13,5 +13,5 @@ type Make() = /// Variables to pass to make target. static member __dispatch__ (context: ActionContext) (variables: Map) = let args = variables |> Seq.fold (fun acc kvp -> $"{acc} {kvp.Key}=\"{kvp.Value}\"") $"{context.Command}" - let ops = [ shellOp "make" args ] - execRequest Cacheability.Always ops + let ops = [ shellOp("make", args) ] + execRequest(Cacheability.Always, ops, false) diff --git a/src/Terrabuild.Extensions/Npm.fs b/src/Terrabuild.Extensions/Npm.fs index a71d881e..3b234d0b 100644 --- a/src/Terrabuild.Extensions/Npm.fs +++ b/src/Terrabuild.Extensions/Npm.fs @@ -34,9 +34,9 @@ type Npm() = let arguments = arguments |> Option.defaultValue "" let ops = [ - shellOp "npm" $"{cmd} {arguments}" + shellOp("npm", $"{cmd} {arguments}") ] - execRequest Cacheability.Always ops + execRequest(Cacheability.Always, ops, false) /// @@ -44,8 +44,8 @@ type Npm() = /// static member install (context: ActionContext) (force: bool option)= let force = if force = Some true then " --force" else "" - let ops = [ shellOp "npm" $"ci{force}" ] - execRequest Cacheability.Always ops + let ops = [ shellOp("npm", $"ci{force}") ] + execRequest(Cacheability.Always, ops, false) /// @@ -56,10 +56,10 @@ type Npm() = let args = arguments |> Option.defaultValue "" let ops = [ - shellOp "npm" "ci" - shellOp "npm" $"run build -- {args}" + shellOp("npm", "ci") + shellOp("npm", $"run build -- {args}") ] - execRequest Cacheability.Always ops + execRequest(Cacheability.Always, ops, false) /// @@ -70,10 +70,10 @@ type Npm() = let args = arguments |> Option.defaultValue "" let ops = [ - shellOp "npm" "ci" - shellOp "npm" $"run test -- {args}" + shellOp("npm", "ci") + shellOp("npm", $"run test -- {args}") ] - execRequest Cacheability.Always ops + execRequest(Cacheability.Always, ops, false) /// /// Run `run` script. @@ -83,6 +83,6 @@ type Npm() = let args = arguments |> Option.defaultValue "" let ops = [ - shellOp "npm" $"run {command} -- {args}" + shellOp("npm", $"run {command} -- {args}") ] - execRequest Cacheability.Always ops + execRequest(Cacheability.Always, ops, false) diff --git a/src/Terrabuild.Extensions/OpenApi.fs b/src/Terrabuild.Extensions/OpenApi.fs index f631edd2..8e3d7427 100644 --- a/src/Terrabuild.Extensions/OpenApi.fs +++ b/src/Terrabuild.Extensions/OpenApi.fs @@ -15,6 +15,6 @@ type OpenApi() = let args = $"generate -i {input} -g {generator} -o {output}" let ops = [ - shellOp "docker-entrypoint.sh" args + shellOp("docker-entrypoint.sh", args) ] - execRequest Cacheability.Always ops + execRequest(Cacheability.Always, ops, false) diff --git a/src/Terrabuild.Extensions/Shell.fs b/src/Terrabuild.Extensions/Shell.fs index 62888cf5..d3846005 100644 --- a/src/Terrabuild.Extensions/Shell.fs +++ b/src/Terrabuild.Extensions/Shell.fs @@ -14,5 +14,5 @@ type Shell() = /// Arguments to pass to command. static member __dispatch__ (context: ActionContext) (arguments: string option) = let arguments = arguments |> Option.defaultValue "" - let ops = [ shellOp context.Command arguments ] - execRequest Cacheability.Always ops + let ops = [ shellOp(context.Command, arguments) ] + execRequest(Cacheability.Always, ops, false) diff --git a/src/Terrabuild.Extensions/Terraform.fs b/src/Terrabuild.Extensions/Terraform.fs index 523437b2..116d1775 100644 --- a/src/Terrabuild.Extensions/Terraform.fs +++ b/src/Terrabuild.Extensions/Terraform.fs @@ -34,8 +34,8 @@ type Terraform() = let arguments = arguments |> Option.defaultValue "" let arguments = $"{context.Command} {arguments}" - let ops = [ shellOp "terraform" arguments ] - execRequest Cacheability.Always ops + let ops = [ shellOp("terraform", arguments) ] + execRequest(Cacheability.Always, ops, true) /// @@ -47,8 +47,8 @@ type Terraform() = match config with | Some config -> $" -backend-config={config}" | _ -> "" - let ops = [ shellOp "terraform" $"init -reconfigure{config}" ] - execRequest Cacheability.Always ops + let ops = [ shellOp("terraform", $"init -reconfigure{config}") ] + execRequest(Cacheability.Always, ops, false) /// @@ -61,15 +61,15 @@ type Terraform() = /// Variables for plan (see Terraform [Variables](https://developer.hashicorp.com/terraform/language/values/variables#variables-on-the-command-line)). static member validate (context: ActionContext) (workspace: string option) = let ops = [ - shellOp "terraform" "init" + shellOp("terraform", "init") match workspace with - | Some workspace -> shellOp "terraform" $"workspace select {workspace}" + | Some workspace -> shellOp("terraform", $"workspace select {workspace}") | _ -> () - shellOp "terraform" "validate" + shellOp("terraform", "validate") ] - execRequest Cacheability.Always ops + execRequest(Cacheability.Always, ops, false) /// @@ -88,15 +88,15 @@ type Terraform() = | _ -> "" let ops = [ - shellOp "terraform" $"init -reconfigure{config}" + shellOp("terraform", $"init -reconfigure{config}") match workspace with - | Some workspace -> shellOp "terraform" $"workspace select {workspace}" + | Some workspace -> shellOp("terraform", $"workspace select {workspace}") | _ -> () - shellOp "terraform" $"plan -out=terrabuild.planfile{vars}" + shellOp("terraform", $"plan -out=terrabuild.planfile{vars}") ] - execRequest Cacheability.Always ops + execRequest(Cacheability.Always, ops, false) /// @@ -113,13 +113,12 @@ type Terraform() = | _ -> "" let ops = [ - shellOp "terraform" $"init -reconfigure{config}" + shellOp("terraform", $"init -reconfigure{config}") match workspace with - | Some workspace -> shellOp "terraform" $"workspace select {workspace}" + | Some workspace -> shellOp("terraform", $"workspace select {workspace}") | _ -> () - shellOp "terraform" "apply -input=false terrabuild.planfile" + shellOp("terraform", "apply -input=false terrabuild.planfile") ] - execRequest Cacheability.Always ops - \ No newline at end of file + execRequest(Cacheability.Always, ops, true) diff --git a/src/Terrabuild.Extensions/Yarn.fs b/src/Terrabuild.Extensions/Yarn.fs index dae27881..849e6a73 100644 --- a/src/Terrabuild.Extensions/Yarn.fs +++ b/src/Terrabuild.Extensions/Yarn.fs @@ -32,9 +32,9 @@ type Yarn() = let cmd = context.Command let ops = [ - shellOp "yarn" $"{cmd} -- {arguments}" + shellOp("yarn", $"{cmd} -- {arguments}") ] - execRequest Cacheability.Always ops + execRequest(Cacheability.Always, ops, false) /// @@ -47,8 +47,8 @@ type Yarn() = | Some true -> " --ignore-engines" | _ -> "" - let ops = [ shellOp "yarn" $"install --frozen-lockfile{ignoreEngines}" ] - execRequest Cacheability.Always ops + let ops = [ shellOp("yarn", $"install --frozen-lockfile{ignoreEngines}") ] + execRequest(Cacheability.Always, ops, false) /// @@ -64,10 +64,10 @@ type Yarn() = | _ -> "" let ops = [ - shellOp "yarn" $"install --frozen-lockfile{ignoreEngines}" - shellOp "yarn" $"build -- {args}" + shellOp("yarn", $"install --frozen-lockfile{ignoreEngines}") + shellOp("yarn", $"build -- {args}") ] - execRequest Cacheability.Always ops + execRequest(Cacheability.Always, ops, false) /// @@ -83,10 +83,10 @@ type Yarn() = | _ -> "" let ops = [ - shellOp "yarn" $"install --frozen-lockfile{ignoreEngines}" - shellOp "yarn" $"test -- {args}" + shellOp("yarn", $"install --frozen-lockfile{ignoreEngines}") + shellOp("yarn", $"test -- {args}") ] - execRequest Cacheability.Always ops + execRequest(Cacheability.Always, ops, false) /// /// Run `run` script. @@ -96,6 +96,6 @@ type Yarn() = let args = arguments |> Option.defaultValue "" let ops = [ - shellOp "yarn" $"{command} -- {args}" + shellOp("yarn", $"{command} -- {args}") ] - execRequest Cacheability.Always ops + execRequest(Cacheability.Always, ops, false) diff --git a/src/Terrabuild/Core/Build.fs b/src/Terrabuild/Core/Build.fs index 84211235..e505fd6b 100644 --- a/src/Terrabuild/Core/Build.fs +++ b/src/Terrabuild/Core/Build.fs @@ -58,8 +58,7 @@ let private containerInfos = Concurrent.ConcurrentDictionary() let buildCommands (node: GraphDef.Node) (options: ConfigOptions.Options) projectDirectory homeDir tmpDir = - node.Operations - |> List.map (fun operation -> + node.Operations |> List.map (fun operation -> let metaCommand = operation.MetaCommand match options.ContainerTool, operation.Container with | Some cmd, Some container -> @@ -124,15 +123,16 @@ let execCommands (node: GraphDef.Node) (cacheEntry: Cache.IEntry) (options: Conf cmdLastEndedAt <- DateTime.UtcNow let endedAt = cmdLastEndedAt let duration = endedAt - startedAt - let stepLog = { Cache.OperationSummary.MetaCommand = metaCommand - Cache.OperationSummary.Command = cmd - Cache.OperationSummary.Arguments = args - Cache.OperationSummary.Container = container - Cache.OperationSummary.StartedAt = startedAt - Cache.OperationSummary.EndedAt = endedAt - Cache.OperationSummary.Duration = duration - Cache.OperationSummary.Log = logFile - Cache.OperationSummary.ExitCode = exitCode } + let stepLog = + { Cache.OperationSummary.MetaCommand = metaCommand + Cache.OperationSummary.Command = cmd + Cache.OperationSummary.Arguments = args + Cache.OperationSummary.Container = container + Cache.OperationSummary.StartedAt = startedAt + Cache.OperationSummary.EndedAt = endedAt + Cache.OperationSummary.Duration = duration + Cache.OperationSummary.Log = logFile + Cache.OperationSummary.ExitCode = exitCode } stepLog |> stepLogs.Add lastStatusCode <- exitCode @@ -166,8 +166,9 @@ let run (options: ConfigOptions.Options) (cache: Cache.ICache) (api: Contracts.I let nodeResults = Concurrent.ConcurrentDictionary() let restorables = Concurrent.ConcurrentDictionary() - let processNode (maxCompletionChildren: DateTime) (node: GraphDef.Node) = - let cacheEntryId = GraphDef.buildCacheKey node + let processNode (node: GraphDef.Node) = + let startedAt = DateTime.UtcNow + notification.NodeBuilding node let projectDirectory = match node.Project with @@ -175,162 +176,144 @@ let run (options: ConfigOptions.Options) (cache: Cache.ICache) (api: Contracts.I | FS.File projectFile -> projectFile |> FS.parentDirectory |> Option.get | _ -> "." - let buildNode() = - let startedAt = DateTime.UtcNow - - notification.NodeBuilding node - - // restore lazy dependencies - node.Dependencies - |> Seq.iter (fun nodeId -> - match restorables.TryGetValue nodeId with - | true, restorable -> restorable.Restore() - | _ -> ()) - - let beforeFiles = - if node.IsLeaf then IO.Snapshot.Empty - else IO.createSnapshot node.Outputs projectDirectory - - let cacheEntry = cache.GetEntry true cacheEntryId - let lastStatusCode, stepLogs = execCommands node cacheEntry options projectDirectory homeDir tmpDir - - // keep only new or modified files - let afterFiles = IO.createSnapshot node.Outputs projectDirectory - let newFiles = afterFiles - beforeFiles - let outputs = IO.copyFiles cacheEntry.Outputs projectDirectory newFiles - - let successful = lastStatusCode = 0 - let endedAt = DateTime.UtcNow - let summary = { Cache.TargetSummary.Project = node.Project - Cache.TargetSummary.Target = node.Target - Cache.TargetSummary.Operations = [ stepLogs |> List.ofSeq ] - Cache.TargetSummary.Outputs = outputs - Cache.TargetSummary.IsSuccessful = successful - Cache.TargetSummary.StartedAt = startedAt - Cache.TargetSummary.EndedAt = endedAt - Cache.TargetSummary.Duration = endedAt - startedAt - Cache.TargetSummary.Cache = node.Cache } - - notification.NodeUploading node - - // create an archive with new files - Log.Debug("{NodeId}: Building '{Project}/{Target}' with {Hash}", node.Id, node.Project, node.Target, node.TargetHash) - let files = cacheEntry.Complete summary - api |> Option.iter (fun api -> api.AddArtifact node.Project node.Target node.ProjectHash node.TargetHash files successful) - - match lastStatusCode with - | 0 -> TaskStatus.Success endedAt - | _ -> TaskStatus.Failure (DateTime.UtcNow, $"{node.Id} failed with exit code {lastStatusCode}") - - let restoreNode () = - notification.NodeScheduled node - let cacheEntryId = GraphDef.buildCacheKey node - match cache.TryGetSummaryOnly allowRemoteCache cacheEntryId with - | Some (_, summary) -> - let dependencies = - node.Dependencies - |> Seq.choose (fun nodeId -> - match restorables.TryGetValue nodeId with - | true, restorable -> Some restorable - | _ -> None) - |> List.ofSeq - - let callback() = - notification.NodeDownloading node - match cache.TryGetSummary allowRemoteCache cacheEntryId with - | Some summary -> - Log.Debug("{NodeId} restoring '{Project}/{Target}' from cache from {Hash}", node.Id, node.Project, node.Target, node.TargetHash) - match summary.Outputs with - | Some outputs -> - let files = IO.enumerateFiles outputs - IO.copyFiles projectDirectory outputs files |> ignore - api |> Option.iter (fun api -> api.UseArtifact node.ProjectHash node.TargetHash) - | _ -> () - notification.NodeCompleted node TaskRequest.Restore true - | _ -> - notification.NodeCompleted node TaskRequest.Restore false - raiseBugError $"Unable to download build output for {cacheEntryId} for node {node.Id}" - - let restorable = Restorable(callback, dependencies) - restorables.TryAdd(node.Id, restorable) |> ignore - if summary.IsSuccessful then TaskStatus.Success summary.EndedAt - else TaskStatus.Failure (summary.EndedAt, $"Restored node {node.Id} with a build in failure state") - | _ -> - TaskStatus.Failure (DateTime.UtcNow, $"Unable to download build output for {cacheEntryId} for node {node.Id}") - - if force then - Log.Debug("{NodeId} must rebuild because force build requested", node.Id) - TaskRequest.Build, buildNode() + // restore lazy dependencies + node.Dependencies |> Seq.iter (fun nodeId -> + match restorables.TryGetValue nodeId with + | true, restorable -> restorable.Restore() + | _ -> ()) - elif maxCompletionChildren = DateTime.MaxValue then - Log.Debug("{NodeId} must rebuild because child is rebuilding", node.Id) - TaskRequest.Build, buildNode() + let beforeFiles = + if node.IsLeaf then IO.Snapshot.Empty + else IO.createSnapshot node.Outputs projectDirectory - elif node.Cache <> Terrabuild.Extensibility.Cacheability.Never then - let cacheEntryId = GraphDef.buildCacheKey node - match cache.TryGetSummaryOnly allowRemoteCache cacheEntryId with - | Some (_, summary) -> - Log.Debug("{NodeId} has existing build summary", node.Id) - - // task is failed and retry requested - if retry && not summary.IsSuccessful then - Log.Debug("{NodeId} must rebuild because node is failed and retry requested", node.Id) - TaskRequest.Build, buildNode() - - // task is cached - else - Log.Debug("{NodeId} is marked as used", node.Id) - TaskRequest.Restore, restoreNode() - | _ -> - Log.Debug("{NodeId} must be build since no summary and required", node.Id) - TaskRequest.Build, buildNode() - else - Log.Debug("{NodeId} is not cacheable", node.Id) - TaskRequest.Build, buildNode() + let cacheEntryId = GraphDef.buildCacheKey node + let cacheEntry = cache.GetEntry true cacheEntryId + let lastStatusCode, stepLogs = execCommands node cacheEntry options projectDirectory homeDir tmpDir + + // keep only new or modified files + let afterFiles = IO.createSnapshot node.Outputs projectDirectory + let newFiles = afterFiles - beforeFiles + let outputs = IO.copyFiles cacheEntry.Outputs projectDirectory newFiles + + let successful = lastStatusCode = 0 + let endedAt = DateTime.UtcNow + let summary = + { Cache.TargetSummary.Project = node.Project + Cache.TargetSummary.Target = node.Target + Cache.TargetSummary.Operations = [ stepLogs |> List.ofSeq ] + Cache.TargetSummary.Outputs = outputs + Cache.TargetSummary.IsSuccessful = successful + Cache.TargetSummary.StartedAt = startedAt + Cache.TargetSummary.EndedAt = endedAt + Cache.TargetSummary.Duration = endedAt - startedAt + Cache.TargetSummary.Cache = node.Cache } + + notification.NodeUploading node + + // create an archive with new files + Log.Debug("{NodeId}: Building '{Project}/{Target}' with {Hash}", node.Id, node.Project, node.Target, node.TargetHash) + let files = cacheEntry.Complete summary + api |> Option.iter (fun api -> api.AddArtifact node.Project node.Target node.ProjectHash node.TargetHash files successful) + + match lastStatusCode with + | 0 -> TaskStatus.Success endedAt + | _ -> TaskStatus.Failure (DateTime.UtcNow, $"{node.Id} failed with exit code {lastStatusCode}") let hub = Hub.Create(options.MaxConcurrency) let rec schedule nodeId = if nodeResults.TryAdd(nodeId, (TaskRequest.Build, TaskStatus.Pending)) then let node = graph.Nodes[nodeId] - let nodeComputed = hub.GetSignal nodeId + notification.NodeScheduled node - // await dependencies - let awaitedDependencies = - node.Dependencies - |> Seq.map (fun awaitedProjectId -> - schedule awaitedProjectId - hub.GetSignal awaitedProjectId) - |> List.ofSeq - - let onAllSignaled () = - try - let maxCompletionChildren = - match awaitedDependencies with - | [ ] -> DateTime.MinValue - | _ -> awaitedDependencies |> Seq.maxBy (fun dep -> dep.Value) |> (fun dep -> dep.Value) - - let buildRequest, completionStatus = processNode maxCompletionChildren node - Log.Debug("{NodeId} completed request {Request} with status {Status}", node.Id, buildRequest, completionStatus) - nodeResults[node.Id] <- (buildRequest, completionStatus) + let projectDirectory = + match node.Project with + | FS.Directory projectDirectory -> projectDirectory + | FS.File projectFile -> projectFile |> FS.parentDirectory |> Option.get + | _ -> "." + let completionStatus = + if force then None + else + let cacheEntryId = GraphDef.buildCacheKey node + match cache.TryGetSummaryOnly allowRemoteCache cacheEntryId with + | Some (_, summary) -> + if retry && not summary.IsSuccessful then None + else + let dependencies = + node.Dependencies |> Seq.choose (fun nodeId -> + match restorables.TryGetValue nodeId with + | true, restorable -> Some restorable + | _ -> None) + |> List.ofSeq + + let callback() = + notification.NodeDownloading node + match cache.TryGetSummary allowRemoteCache cacheEntryId with + | Some summary -> + Log.Debug("{NodeId} restoring '{Project}/{Target}' with {Hash}", node.Id, node.Project, node.Target, node.TargetHash) + match summary.Outputs with + | Some outputs -> + let files = IO.enumerateFiles outputs + IO.copyFiles projectDirectory outputs files |> ignore + api |> Option.iter (fun api -> api.UseArtifact node.ProjectHash node.TargetHash) + | _ -> () + notification.NodeCompleted node TaskRequest.Restore true + | _ -> + notification.NodeCompleted node TaskRequest.Restore false + raiseBugError $"Unable to download build output for {cacheEntryId} for node {node.Id}" + + if node.Managed then + let restorable = Restorable(callback, dependencies) + restorables.TryAdd(node.Id, restorable) |> ignore + else + Log.Debug("{NodeId} skipping restore unmanaged '{Project}/{Target}' with {Hash}", node.Id, node.Project, node.Target, node.TargetHash) + notification.NodeCompleted node TaskRequest.Restore true + if summary.IsSuccessful then TaskStatus.Success summary.EndedAt |> Some + else TaskStatus.Failure (summary.EndedAt, $"Restored node {node.Id} with a build in failure state") |> Some + | _ -> None + + let nodeComputed = hub.GetSignal nodeId + match completionStatus with + | Some completionStatus -> + Log.Debug("{NodeId} completed restore request with status {Status}", node.Id, completionStatus) + nodeResults[node.Id] <- (TaskRequest.Restore, completionStatus) + let success, completionDate = match completionStatus with - | TaskStatus.Success completionDate -> - nodeComputed.Value <- completionDate - notification.NodeCompleted node buildRequest true - | _ -> - notification.NodeCompleted node buildRequest false - with - exn -> - Log.Fatal(exn, "{NodeId} unexpected failure while building", node.Id) + | TaskStatus.Success completionDate -> true, completionDate + | TaskStatus.Failure (completionDate, _) -> false, completionDate + | _ -> raiseBugError "Unexpected pending state" + notification.NodeCompleted node TaskRequest.Restore success + if success then nodeComputed.Value <- completionDate + | _ -> + // await dependencies + let awaitedDependencies = + node.Dependencies |> Seq.map (fun awaitedProjectId -> + schedule awaitedProjectId + hub.GetSignal awaitedProjectId) + |> List.ofSeq + let onAllSignaled () = + try + let completionStatus = processNode node + Log.Debug("{NodeId} completed build request with status {Status}", node.Id, completionStatus) + nodeResults[node.Id] <- (TaskRequest.Build, completionStatus) + let success, completionDate = + match completionStatus with + | TaskStatus.Success completionDate -> true, completionDate + | TaskStatus.Failure (completionDate, _) -> false, completionDate + | _ -> raiseBugError "Unexpected pending state" + notification.NodeCompleted node TaskRequest.Build success + if success then nodeComputed.Value <- completionDate + with + | exn -> + Log.Fatal(exn, "{NodeId} failed on build request", node.Id) nodeResults[node.Id] <- (TaskRequest.Build, TaskStatus.Failure (DateTime.UtcNow, exn.Message)) notification.NodeCompleted node TaskRequest.Build false - reraise() - let awaitedSignals = awaitedDependencies |> List.map (fun entry -> entry :> ISignal) - hub.Subscribe nodeId awaitedSignals onAllSignaled + let awaitedSignals = awaitedDependencies |> List.map (fun entry -> entry :> ISignal) + hub.Subscribe nodeId awaitedSignals onAllSignaled graph.RootNodes |> Seq.iter schedule @@ -363,19 +346,19 @@ let run (options: ConfigOptions.Options) (cache: Cache.ICache) (api: Contracts.I |> Map.choose getDependencyStatus let isSuccess = - graph.RootNodes - |> Set.forall (fun nodeId -> + graph.RootNodes |> Set.forall (fun nodeId -> match nodeStatus |> Map.tryFind nodeId with | Some info -> info.Status.IsSuccess | _ -> false) - let buildInfo = { Summary.Commit = headCommit.Sha - Summary.BranchOrTag = branchOrTag - Summary.StartedAt = startedAt - Summary.EndedAt = DateTime.UtcNow - Summary.IsSuccess = isSuccess - Summary.Targets = options.Targets - Summary.Nodes = nodeStatus } + let buildInfo = + { Summary.Commit = headCommit.Sha + Summary.BranchOrTag = branchOrTag + Summary.StartedAt = startedAt + Summary.EndedAt = DateTime.UtcNow + Summary.IsSuccess = isSuccess + Summary.Targets = options.Targets + Summary.Nodes = nodeStatus } notification.BuildCompleted buildInfo api |> Option.iter (fun api -> api.CompleteBuild buildInfo.IsSuccess) @@ -407,13 +390,10 @@ let loadSummary (options: ConfigOptions.Options) (cache: Cache.ICache) (graph: G NodeInfo.TargetHash = node.TargetHash } |> Some | _ -> None - - graph.Nodes - |> Map.choose getDependencyStatus + graph.Nodes |> Map.choose getDependencyStatus let isSuccess = - graph.RootNodes - |> Set.forall (fun nodeId -> + graph.RootNodes |> Set.forall (fun nodeId -> match nodeStatus |> Map.tryFind nodeId with | Some info -> info.Status.IsSuccess | _ -> false) diff --git a/src/Terrabuild/Core/Builder.fs b/src/Terrabuild/Core/Builder.fs index d8df0336..0e0c0631 100644 --- a/src/Terrabuild/Core/Builder.fs +++ b/src/Terrabuild/Core/Builder.fs @@ -45,8 +45,7 @@ let build (options: ConfigOptions.Options) (configuration: Configuration.Workspa // apply on each dependency let inChildren, outChildren = - dependsOns - |> Set.fold (fun (accInChildren, accOutChildren) dependsOn -> + dependsOns |> Set.fold (fun (accInChildren, accOutChildren) dependsOn -> match dependsOn with | String.Regex "^\^(.+)$" [ parentDependsOn ] -> accInChildren, accOutChildren + projectConfig.Dependencies |> Set.collect (buildTarget parentDependsOn) @@ -61,9 +60,8 @@ let build (options: ConfigOptions.Options) (configuration: Configuration.Workspa // barrier nodes are just discarded and dependencies lift level up match projectConfig.Targets |> Map.tryFind targetName with | Some target -> - let cache, ops = - target.Operations - |> List.fold (fun (cache, ops) operation -> + let cache, sideEffect, ops = + target.Operations |> List.fold (fun (cache, sideEffect, ops) operation -> let optContext = { Terrabuild.Extensibility.ActionContext.Debug = options.Debug Terrabuild.Extensibility.ActionContext.CI = options.Run.IsSome @@ -88,8 +86,7 @@ let build (options: ConfigOptions.Options) (configuration: Configuration.Workspa | _ -> raiseExternalError $"{hash}: Failed to get shell operation (extension error)" let newops = - executionRequest.Operations - |> List.map (fun shellOperation -> { + executionRequest.Operations |> List.map (fun shellOperation -> { ContaineredShellOperation.Container = operation.Container ContaineredShellOperation.ContainerPlatform = operation.Platform ContaineredShellOperation.ContainerVariables = operation.ContainerVariables @@ -98,8 +95,9 @@ let build (options: ConfigOptions.Options) (configuration: Configuration.Workspa ContaineredShellOperation.Arguments = shellOperation.Arguments }) let cache = cache &&& executionRequest.Cache - cache, ops @ newops - ) (Cacheability.Always, []) + let sideEffect = sideEffect || executionRequest.SideEffect + cache, sideEffect, ops @ newops + ) (Cacheability.Always, false, []) let opsCmds = ops @@ -121,6 +119,12 @@ let build (options: ConfigOptions.Options) (configuration: Configuration.Workspa elif options.LocalOnly then Cacheability.Local else target.Cache |> Option.defaultValue cache + let managed = target.Managed |> Option.defaultValue true + + let targetOutput = + if managed then target.Outputs + else Set.empty + let node = { Node.Id = nodeId Node.Label = $"{targetName} {projectConfig.Name}" @@ -129,9 +133,10 @@ let build (options: ConfigOptions.Options) (configuration: Configuration.Workspa Node.ConfigurationTarget = target Node.Operations = ops Node.Cache = cache + Node.Managed = managed Node.Dependencies = children - Node.Outputs = target.Outputs + Node.Outputs = targetOutput Node.ProjectHash = projectConfig.Hash Node.TargetHash = hash @@ -152,10 +157,9 @@ let build (options: ConfigOptions.Options) (configuration: Configuration.Workspa node2children[nodeId] let rootNodes = - configuration.SelectedProjects - |> Seq.collect (fun dependency -> options.Targets - |> Seq.collect (fun target -> - buildTarget target dependency)) + configuration.SelectedProjects |> Seq.collect (fun dependency -> + options.Targets |> Seq.collect (fun target -> + buildTarget target dependency)) |> Set let endedAt = DateTime.UtcNow diff --git a/src/Terrabuild/Core/Configuration.fs b/src/Terrabuild/Core/Configuration.fs index a2a129c4..0c63b661 100644 --- a/src/Terrabuild/Core/Configuration.fs +++ b/src/Terrabuild/Core/Configuration.fs @@ -31,6 +31,7 @@ type Target = { DependsOn: string set Outputs: string set Cache: Cacheability option + Managed: bool option Operations: TargetOperation list } @@ -549,11 +550,16 @@ let private finalizeProject projectDir evaluationContext (projectDef: LoadedProj | None -> None | _ -> raiseParseError "invalid cache value" + let targetManaged = + target.Managed + |> Option.bind (Eval.asBoolOption << Eval.eval evaluationContext) + let target = { Target.Hash = hash Target.Rebuild = rebuild Target.DependsOn = dependsOn Target.Cache = targetCache + Target.Managed = targetManaged Target.Outputs = outputs Target.Operations = targetOperations } diff --git a/src/Terrabuild/Core/GraphDef.fs b/src/Terrabuild/Core/GraphDef.fs index 2c4115b8..6da6ca66 100644 --- a/src/Terrabuild/Core/GraphDef.fs +++ b/src/Terrabuild/Core/GraphDef.fs @@ -27,6 +27,7 @@ type Node = { TargetHash: string Operations: ContaineredShellOperation list Cache: Terrabuild.Extensibility.Cacheability + Managed: bool // tell if a node is leaf (that is no dependencies in same project) IsLeaf: bool diff --git a/tests/cluster-layers/results/terrabuild-debug.build-graph.json b/tests/cluster-layers/results/terrabuild-debug.build-graph.json index 0b9e42e0..af102623 100644 --- a/tests/cluster-layers/results/terrabuild-debug.build-graph.json +++ b/tests/cluster-layers/results/terrabuild-debug.build-graph.json @@ -77,6 +77,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "b:build": { @@ -156,6 +157,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "c:build": { @@ -235,6 +237,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "d:build": { @@ -294,6 +297,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "e:build": { @@ -353,6 +357,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "f:build": { @@ -410,6 +415,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "g:build": { @@ -466,6 +472,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true } }, diff --git a/tests/multirefs/results/terrabuild-debug.build-graph.json b/tests/multirefs/results/terrabuild-debug.build-graph.json index d476dfd9..949cd48b 100644 --- a/tests/multirefs/results/terrabuild-debug.build-graph.json +++ b/tests/multirefs/results/terrabuild-debug.build-graph.json @@ -47,6 +47,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "b:build": { @@ -95,6 +96,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "c:build": { @@ -141,6 +143,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true } }, diff --git a/tests/simple/results/terrabuild-debug.build-graph.json b/tests/simple/results/terrabuild-debug.build-graph.json index 0fe253d5..e322f52d 100644 --- a/tests/simple/results/terrabuild-debug.build-graph.json +++ b/tests/simple/results/terrabuild-debug.build-graph.json @@ -84,6 +84,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "libraries/dotnet-lib:build": { @@ -148,6 +149,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "libraries/npm-lib:build": { @@ -205,6 +207,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "libraries/shell-lib:build": { @@ -251,6 +254,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "projects/dotnet-app:build": { @@ -317,6 +321,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "projects/make-app:build": { @@ -396,6 +401,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "projects/npm-app/private-npm-lib:build": { @@ -453,6 +459,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "projects/npm-app:build": { @@ -513,6 +520,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "projects/open-api:build": { @@ -571,6 +579,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true }, "projects/rust-app:build": { @@ -622,6 +631,7 @@ } ], "cache": 0, + "managed": true, "isLeaf": true } },