-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathlineage.go
More file actions
144 lines (128 loc) · 4.22 KB
/
Copy pathlineage.go
File metadata and controls
144 lines (128 loc) · 4.22 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
package main
import (
"fmt"
"time"
)
// LineageEntry records one step in a model's derivation chain.
type LineageEntry struct {
Step int `json:"step" yaml:"step"`
Action string `json:"action" yaml:"action"` // download, quantize, fine-tune, merge, promote
Description string `json:"description" yaml:"description"`
InputHash string `json:"input_hash,omitempty" yaml:"input_hash,omitempty"`
OutputHash string `json:"output_hash,omitempty" yaml:"output_hash,omitempty"`
Tool string `json:"tool,omitempty" yaml:"tool,omitempty"`
Timestamp string `json:"timestamp" yaml:"timestamp"`
Actor string `json:"actor,omitempty" yaml:"actor,omitempty"`
Metadata map[string]string `json:"metadata,omitempty" yaml:"metadata,omitempty"`
}
// LineageChain is the full derivation history for a model artifact.
type LineageChain struct {
ArtifactName string `json:"artifact_name" yaml:"artifact_name"`
ArtifactHash string `json:"artifact_hash" yaml:"artifact_hash"`
Entries []LineageEntry `json:"entries" yaml:"entries"`
}
// NewLineageChain starts a lineage chain for a model artifact.
func NewLineageChain(name, hash string) *LineageChain {
return &LineageChain{
ArtifactName: name,
ArtifactHash: hash,
}
}
// AddEntry appends a lineage step.
func (lc *LineageChain) AddEntry(action, description, inputHash, outputHash, tool, actor string, meta map[string]string) {
lc.Entries = append(lc.Entries, LineageEntry{
Step: len(lc.Entries) + 1,
Action: action,
Description: description,
InputHash: inputHash,
OutputHash: outputHash,
Tool: tool,
Timestamp: time.Now().UTC().Format(time.RFC3339),
Actor: actor,
Metadata: meta,
})
}
// BuildQuantizationLineage creates lineage entries for a quantized model.
func BuildQuantizationLineage(baseModel, baseHash, quantMethod, outputHash, quantizer string) *LineageChain {
chain := NewLineageChain(baseModel+"."+quantMethod, outputHash)
chain.AddEntry(
"download",
fmt.Sprintf("base model obtained: %s", baseModel),
"", baseHash, "", "operator", nil,
)
chain.AddEntry(
"quantize",
fmt.Sprintf("quantized with method %s", quantMethod),
baseHash, outputHash, "llama.cpp/quantize",
quantizer,
map[string]string{
"method": quantMethod,
"base_model": baseModel,
},
)
return chain
}
// BuildAdapterLineage creates lineage for a model with applied adapters.
func BuildAdapterLineage(baseModel, baseHash string, adapters []Adapter, finalHash string) *LineageChain {
chain := NewLineageChain(baseModel+"+adapters", finalHash)
chain.AddEntry(
"download",
fmt.Sprintf("base model: %s", baseModel),
"", baseHash, "", "operator", nil,
)
prevHash := baseHash
for _, adapter := range adapters {
outHash := adapter.Hash
if outHash == "" {
outHash = finalHash
}
chain.AddEntry(
"fine-tune",
fmt.Sprintf("applied adapter: %s (type=%s)", adapter.Name, adapter.Type),
prevHash, outHash, adapter.Type,
"",
map[string]string{
"adapter_name": adapter.Name,
"adapter_type": adapter.Type,
},
)
prevHash = outHash
}
return chain
}
// BuildPromotionLineage adds a promotion step (quarantine → approved).
func (lc *LineageChain) AddPromotion(status, actor, reason string) {
lc.AddEntry(
"promote",
fmt.Sprintf("promotion to %s: %s", status, reason),
lc.ArtifactHash, lc.ArtifactHash, "ai-model-registry",
actor,
map[string]string{
"promotion_status": status,
"reason": reason,
},
)
}
// ToProperties converts lineage into CycloneDX properties for embedding in a component.
func (lc *LineageChain) ToProperties() []Property {
var props []Property
for _, e := range lc.Entries {
props = append(props, Property{
Name: fmt.Sprintf("secai:lineage:step%d:%s", e.Step, e.Action),
Value: e.Description,
})
}
return props
}
// Verify checks that the hash chain is internally consistent
// (each step's output_hash matches the next step's input_hash).
func (lc *LineageChain) Verify() (bool, int) {
for i := 1; i < len(lc.Entries); i++ {
prev := lc.Entries[i-1]
curr := lc.Entries[i]
if prev.OutputHash != "" && curr.InputHash != "" && prev.OutputHash != curr.InputHash {
return false, i
}
}
return true, -1
}