feat: managed artifacts

.github/workflows/on-push-branch.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Build Self
         run: |
           echo "# Self Build" >> $GITHUB_STEP_SUMMARY
-          make self config=Release terrabuild=$PWD/.out/dotnet/terrabuild
+          make self config=Release
 
       - name: Self Integration Tests
         run: make smoke-tests terrabuild=$PWD/.out/dotnet/terrabuild

Makefile

@@ -73,10 +73,10 @@ docs:
 	dotnet run --project tools/DocGen -- src/Terrabuild.Extensions/bin/$(config)/net9.0/Terrabuild.Extensions.xml ../terrabuild.io/content/docs/extensions
 
 self: clean publish
-	$(terrabuild) run build test dist --configuration $(config) --retry --debug --log --local-only
+	$(PWD)/.out/dotnet/terrabuild run build test dist --configuration $(config) --debug --log --retry --local-only
 
 self-logs:
-	$(terrabuild) logs build test dist --configuration $(config) --debug --log --local-only
+	$(PWD)/.out/dotnet/terrabuild logs build test dist --configuration $(config) --debug --log --local-only
 
 logs:
 	$(terrabuild) logs build test dist --configuration $(config) --debug --log --local-only

src/Terrabuild.Configuration.Tests/Project.fs

@@ -47,19 +47,22 @@ let parseProject() =
               TargetBlock.Rebuild = None
               TargetBlock.Outputs = None
               TargetBlock.Cache = None
+              TargetBlock.Managed = None
               TargetBlock.Steps = [ { Extension = "@dotnet"; Command = "build"; Parameters = Map.empty } ] }
         let targetDist =
             { TargetBlock.DependsOn = None
               TargetBlock.Rebuild = None
               TargetBlock.Outputs = None
               TargetBlock.Cache = None
+              TargetBlock.Managed = Expr.False |> Some
               TargetBlock.Steps = [ { Extension = "@dotnet"; Command = "build"; Parameters = Map.empty }
                                     { Extension = "@dotnet"; Command = "publish"; Parameters = Map.empty } ] }
         let targetDocker =
             { TargetBlock.DependsOn = None
               TargetBlock.Rebuild = Some (Expr.Bool false)
               TargetBlock.Outputs = None
               TargetBlock.Cache = "always" |> Expr.String |> Some
+              TargetBlock.Managed = None
               TargetBlock.Steps = [ { Extension = "@shell"; Command = "echo"
                                       Parameters = Map [ "arguments", Expr.Function (Function.Trim,
                                                                                      [ Expr.Function (Function.Plus,
@@ -115,6 +118,7 @@ let parseProject2() =
                                                                  Expr.String ".dll" ])] |> Some
               TargetBlock.DependsOn = None
               TargetBlock.Cache = None
+              TargetBlock.Managed = None
               TargetBlock.Steps = [ { Extension = "@dotnet"; Command = "build"; Parameters = Map.empty } ] }
 
         let locals = 

src/Terrabuild.Configuration.Tests/TestFiles/Success_PROJECT

@@ -37,6 +37,7 @@ target build {
 }
 
 target dist {
+  managed = false
   @dotnet build { }
   @dotnet publish { }
 }

src/Terrabuild.Configuration/AST/Project.fs

@@ -26,6 +26,7 @@ type TargetBlock =
       Outputs: Expr option
       DependsOn: Set<string> option
       Cache: Expr option
+      Managed: Expr option
       Steps: Step list }
 
 [<RequireQualifiedAccess>]

src/Terrabuild.Configuration/Transpiler/Project.fs

@@ -71,7 +71,7 @@ let toProject (block: Block) =
 
 let toTarget (block: Block) =
     block
-    |> checkAllowedAttributes ["rebuild"; "outputs"; "depends_on"; "cache"]
+    |> checkAllowedAttributes ["rebuild"; "outputs"; "depends_on"; "cache"; "managed"]
     |> ignore
 
     let rebuild = block |> tryFindAttribute "rebuild"
@@ -85,6 +85,7 @@ let toTarget (block: Block) =
                 | String.Regex "^target\.(.*)$" [targetIdentifier] -> targetIdentifier
                 | _ -> raiseInvalidArg $"Invalid target dependency '{dependency}'"))
     let cache = block |> tryFindAttribute "cache"
+    let managed = block |> tryFindAttribute "managed"
     let steps =
         block.Blocks
         |> List.map (fun step ->
@@ -110,7 +111,8 @@ let toTarget (block: Block) =
       TargetBlock.Outputs = outputs
       TargetBlock.DependsOn = dependsOn
       TargetBlock.Cache = cache
-      TargetBlock.Steps = steps }
+      TargetBlock.Steps = steps
+      TargetBlock.Managed = managed }
 
 
 

src/Terrabuild.Extensibility/Extensions.fs

@@ -50,13 +50,15 @@ type ShellOperations = ShellOperation list
 type ActionExecutionRequest = {
     Cache: Cacheability
     Operations: ShellOperations
+    SideEffect: bool
 }
 
 
-let shellOp cmd args = 
+let shellOp(cmd, args) = 
     { ShellOperation.Command = cmd
       ShellOperation.Arguments = args }
 
-let execRequest cache ops =
+let execRequest(cache, ops, sideEffect) =
     { ActionExecutionRequest.Cache = cache 
-      ActionExecutionRequest.Operations = ops }
+      ActionExecutionRequest.Operations = ops
+      ActionExecutionRequest.SideEffect = sideEffect }

src/Terrabuild.Extensions/Cargo.fs

@@ -47,8 +47,8 @@ type Cargo() =
         let arguments = arguments |> Option.defaultValue ""
         let arguments = $"{context.Command} {arguments}"
 
-        let ops = [ shellOp "cargo" arguments ]
-        execRequest Cacheability.Always ops
+        let ops = [ shellOp("cargo", arguments) ]
+        execRequest(Cacheability.Always, ops, false)
 
 
     /// <summary title="Build project.">
@@ -60,8 +60,8 @@ type Cargo() =
         let profile = profile |> Option.defaultValue "dev"
         let arguments = arguments |> Option.defaultValue ""
 
-        let ops = [ shellOp "cargo" $"build --profile {profile} {arguments}" ]
-        execRequest Cacheability.Always ops
+        let ops = [ shellOp("cargo", $"build --profile {profile} {arguments}") ]
+        execRequest(Cacheability.Always, ops, false)
 
 
     /// <summary>
@@ -73,5 +73,5 @@ type Cargo() =
         let profile = profile |> Option.defaultValue "dev"
         let arguments = arguments |> Option.defaultValue ""
 
-        let ops = [ shellOp "cargo" $"test --profile {profile} {arguments}" ]
-        execRequest Cacheability.Always ops
+        let ops = [ shellOp("cargo", $"test --profile {profile} {arguments}") ]
+        execRequest(Cacheability.Always, ops, false)

src/Terrabuild.Extensions/Docker.fs

@@ -16,8 +16,8 @@ type Docker() =
         let arguments = arguments |> Option.defaultValue ""
         let arguments = $"{context.Command} {arguments}"
 
-        let ops = [ shellOp "docker" arguments ]
-        execRequest Cacheability.Always ops
+        let ops = [ shellOp("docker", arguments) ]
+        execRequest(Cacheability.Always, ops, true)
 
 
     /// <summary>
@@ -42,15 +42,15 @@ type Docker() =
         let ops = 
             [
                 let buildArgs = $"build --file {dockerfile} --tag {image}:{context.Hash}{args}{platformArgs} ."
-                shellOp "docker" buildArgs
-                if context.CI then shellOp "docker" $"push {image}:{context.Hash}"
+                shellOp("docker", buildArgs)
+                if context.CI then shellOp("docker", $"push {image}:{context.Hash}")
             ]
 
         let cacheability =
             if context.CI then Cacheability.Remote
             else Cacheability.Local
 
-        execRequest cacheability ops
+        execRequest(cacheability, ops, true)
 
 
     /// <summary>
@@ -62,13 +62,13 @@ type Docker() =
         let ops =
             [
                 if context.CI then
-                    shellOp "docker" $"buildx imagetools create -t {image}:{tag} {image}:{context.Hash}"
+                    shellOp("docker", $"buildx imagetools create -t {image}:{tag} {image}:{context.Hash}")
                 else
-                    shellOp "docker" $"tag {image}:{context.Hash} {image}:{tag}"
+                    shellOp("docker", $"tag {image}:{context.Hash} {image}:{tag}")
             ]
 
         let cacheability =
             if context.CI then Cacheability.Remote
             else Cacheability.Local
 
-        execRequest cacheability ops
+        execRequest(cacheability, ops, true)

src/Terrabuild.Extensions/Dotnet.fs

@@ -56,9 +56,6 @@ module DotnetHelpers =
 /// </summary>
 type Dotnet() =
 
-    static let buildRequest (context: ActionContext) buildOps =
-        execRequest Cacheability.Always buildOps
-
     /// <summary>
     /// Provides default values for project.
     /// </summary>
@@ -85,8 +82,8 @@ type Dotnet() =
         let arguments = arguments |> Option.defaultValue ""
         let arguments = $"{context.Command} {arguments}"
 
-        let ops = [ shellOp "dotnet" arguments ]
-        execRequest Cacheability.Always ops
+        let ops = [ shellOp("dotnet", arguments) ]
+        execRequest(Cacheability.Always, ops, false)
 
 
     /// <summary title="Build project.">
@@ -118,11 +115,11 @@ type Dotnet() =
 
         let arguments = arguments |> Option.defaultValue ""
 
-        let buildOps = [
-            shellOp "dotnet" $"build --no-dependencies --configuration {configuration} {logger} {maxcpucount} {version} {arguments}"
+        let ops = [
+            shellOp("dotnet", $"build --no-dependencies --configuration {configuration} {logger} {maxcpucount} {version} {arguments}")
         ]
 
-        buildRequest context buildOps
+        execRequest(Cacheability.Always, ops, false)
 
 
     /// <summary>
@@ -136,11 +133,11 @@ type Dotnet() =
         let version = version |> Option.defaultValue "0.0.0"
         let arguments = arguments |> Option.defaultValue ""
 
-        let buildOps = [
-            shellOp "dotnet" $"pack --no-build --configuration {configuration} /p:Version={version} /p:TargetsForTfmSpecificContentInPackage= {arguments}"
+        let ops = [
+            shellOp("dotnet", $"pack --no-build --configuration {configuration} /p:Version={version} /p:TargetsForTfmSpecificContentInPackage= {arguments}")
         ]
 
-        buildRequest context buildOps
+        execRequest(Cacheability.Always, ops, false)
 
     /// <summary>
     /// Publish a project.
@@ -167,11 +164,11 @@ type Dotnet() =
             | _ -> ""
         let arguments = arguments |> Option.defaultValue ""
 
-        let buildOps = [
-            shellOp "dotnet" $"publish --no-dependencies --configuration {configuration} {runtime} {trim} {single} {arguments}"
+        let ops = [
+            shellOp("dotnet", $"publish --no-dependencies --configuration {configuration} {runtime} {trim} {single} {arguments}")
         ]
 
-        buildRequest context buildOps
+        execRequest(Cacheability.Always, ops, false)
 
     /// <summary>
     /// Restore packages.
@@ -181,8 +178,8 @@ type Dotnet() =
     static member restore (arguments: string option) =
         let arguments = arguments |> Option.defaultValue ""
 
-        let ops = [ shellOp "dotnet" $"restore {arguments}" ]
-        execRequest Cacheability.Local ops
+        let ops = [ shellOp( "dotnet", $"restore {arguments}") ]
+        execRequest(Cacheability.Local, ops, false)
 
 
     /// <summary>
@@ -196,8 +193,8 @@ type Dotnet() =
         let filter = filter |> Option.map (fun filter -> $" --filter \"{filter}\"") |> Option.defaultValue ""
         let arguments = arguments |> Option.defaultValue ""
 
-        let buildOps = [
-            shellOp "dotnet" $"test --no-build --configuration {configuration} {filter} {arguments}"
+        let ops = [
+            shellOp("dotnet", $"test --no-build --configuration {configuration} {filter} {arguments}")
         ]
 
-        buildRequest context buildOps
+        execRequest(Cacheability.Local, ops, false)

src/Terrabuild.Extensions/Gradle.fs

@@ -30,8 +30,8 @@ type Gradle() =
         let arguments = arguments |> Option.defaultValue ""
         let arguments = $"{context.Command} {arguments}"
 
-        let ops = [ shellOp "gradle" arguments ]
-        execRequest Cacheability.Always ops
+        let ops = [ shellOp("gradle", arguments) ]
+        execRequest(Cacheability.Always, ops, false)
 
 
     /// <summary>
@@ -41,6 +41,5 @@ type Gradle() =
     static member build (context: ActionContext) (configuration: string option) =
         let configuration = configuration |> Option.defaultValue GradleHelpers.defaultConfiguration
 
-        let ops = [ shellOp "gradlew" $"assemble{configuration}" ]
-
-        execRequest Cacheability.Always ops
+        let ops = [ shellOp("gradlew", $"assemble{configuration}") ]
+        execRequest(Cacheability.Always, ops, false)

src/Terrabuild.Extensions/Make.fs

@@ -13,5 +13,5 @@ type Make() =
     /// <param name="variables" example="{ configuration: &quot;Release&quot; }">Variables to pass to make target.</param>
     static member __dispatch__ (context: ActionContext) (variables: Map<string, string>) =
         let args = variables |> Seq.fold (fun acc kvp -> $"{acc} {kvp.Key}=\"{kvp.Value}\"") $"{context.Command}"
-        let ops = [ shellOp "make" args ]
-        execRequest Cacheability.Always ops
+        let ops = [ shellOp("make", args) ]
+        execRequest(Cacheability.Always, ops, false)

src/Terrabuild.Extensions/Npm.fs

@@ -34,18 +34,18 @@ type Npm() =
         let arguments = arguments |> Option.defaultValue ""
 
         let ops = [
-            shellOp "npm" $"{cmd} {arguments}"
+            shellOp("npm", $"{cmd} {arguments}")
         ]
-        execRequest Cacheability.Always ops
+        execRequest(Cacheability.Always, ops, false)
 
 
     /// <summary>
     /// Install packages using lock file.
     /// </summary>
     static member install (context: ActionContext) (force: bool option)=
         let force = if force = Some true then " --force" else ""
-        let ops = [ shellOp "npm" $"ci{force}" ]
-        execRequest Cacheability.Always ops
+        let ops = [ shellOp("npm", $"ci{force}") ]
+        execRequest(Cacheability.Always, ops, false)
 
 
     /// <summary>
@@ -56,10 +56,10 @@ type Npm() =
         let args = arguments |> Option.defaultValue ""
 
         let ops = [
-            shellOp "npm" "ci"
-            shellOp "npm" $"run build -- {args}"   
+            shellOp("npm", "ci")
+            shellOp("npm", $"run build -- {args}")   
         ]
-        execRequest Cacheability.Always ops
+        execRequest(Cacheability.Always, ops, false)
 
 
     /// <summary>
@@ -70,10 +70,10 @@ type Npm() =
         let args = arguments |> Option.defaultValue ""
 
         let ops = [
-            shellOp "npm" "ci"
-            shellOp "npm" $"run test -- {args}"   
+            shellOp("npm", "ci")
+            shellOp("npm", $"run test -- {args}")   
         ]
-        execRequest Cacheability.Always ops
+        execRequest(Cacheability.Always, ops, false)
 
     /// <summary>
     /// Run `run` script.
@@ -83,6 +83,6 @@ type Npm() =
         let args = arguments |> Option.defaultValue ""
 
         let ops = [
-            shellOp "npm" $"run {command} -- {args}"
+            shellOp("npm", $"run {command} -- {args}")
         ]
-        execRequest Cacheability.Always ops
+        execRequest(Cacheability.Always, ops, false)

src/Terrabuild.Extensions/OpenApi.fs

@@ -15,6 +15,6 @@ type OpenApi() =
         let args = $"generate -i {input} -g {generator} -o {output}"
 
         let ops = [
-            shellOp "docker-entrypoint.sh" args
+            shellOp("docker-entrypoint.sh", args)
         ]
-        execRequest Cacheability.Always ops
+        execRequest(Cacheability.Always, ops, false)

src/Terrabuild.Extensions/Shell.fs

@@ -14,5 +14,5 @@ type Shell() =
     /// <param name="arguments" example="&quot;Hello Terrabuild&quot;">Arguments to pass to command.</param>
     static member __dispatch__ (context: ActionContext) (arguments: string option) =
         let arguments = arguments |> Option.defaultValue ""
-        let ops = [ shellOp context.Command arguments ]
-        execRequest Cacheability.Always ops
+        let ops = [ shellOp(context.Command, arguments) ]
+        execRequest(Cacheability.Always, ops, false)