Per-stem melody + stems dB levels + meter (levels 0.72 -> 0.79 on the new scorer)

musicstate-generator/src/musicstate/analyzers/__init__.py

@@ -13,6 +13,7 @@
 from .notes import NoteTranscriptionAnalyzer
 from .pump import PumpAnalyzer
 from .semantic import SemanticAnalyzer
+from .stem_separation import StemSeparationAnalyzer
 from .stems import StemsAnalyzer
 from .structure import StructureAnalyzer
 
@@ -31,6 +32,7 @@
     "MelodyAnalyzer",
     "NoteTranscriptionAnalyzer",
     "Allin1Analyzer",
+    "StemSeparationAnalyzer",
     "StemsAnalyzer",
     "SemanticAnalyzer",
     "EmbeddingAnalyzer",

musicstate-generator/src/musicstate/analyzers/melody.py

@@ -38,6 +38,51 @@ def __init__(self, hop_length: int = HOP_LENGTH):
         self.hop_length = hop_length
 
     def analyze(self, audio, sample_rate, ctx):
+        vocals = (ctx.get("stems_audio") or {}).get("vocals")
+        if vocals is not None:
+            return self._from_vocals(vocals, sample_rate)
+        return self._from_mix(audio, sample_rate, ctx)
+
+    def _from_vocals(self, vocals, sample_rate):
+        """Pitch-track the isolated vocals stem into discrete note events. On the mix
+        the dominant pitch is rarely the vocal; on the stem it is."""
+        hop = 256
+        f0, voiced, _ = librosa.pyin(
+            np.ascontiguousarray(vocals), sr=sample_rate, hop_length=hop,
+            frame_length=2048, fmin=110.0, fmax=1000.0)
+        times = librosa.times_like(f0, sr=sample_rate, hop_length=hop)
+
+        def midi_of(hz):
+            return int(round(69 + 12 * np.log2(hz / 440.0)))
+
+        notes, i, n = [], 0, len(f0)
+        while i < n:
+            if not voiced[i] or not np.isfinite(f0[i]) or f0[i] <= 0:
+                i += 1
+                continue
+            m = midi_of(f0[i])
+            j = i
+            while (j < n and voiced[j] and np.isfinite(f0[j]) and f0[j] > 0
+                   and midi_of(f0[j]) == m):
+                j += 1
+            dur = float(times[min(j, n - 1)] - times[i])
+            if dur >= 0.05:
+                notes.append([round(float(times[i]), 3), round(dur, 3), m,
+                              _NOTES[m % 12] + str(m // 12 - 1)])
+            i = j
+
+        return AnalyzerResult(
+            status="ok" if len(notes) >= 20 else "not_computed",
+            patch={"melody": {
+                "rate": "per_note", "of": "vocals",
+                "how": "pyin f0 on the isolated vocals stem, notes segmented by semitone",
+                "unit": "[start_s, dur_s, midi, name]", "notes": notes,
+            }},
+            confidence={"melody": 0.75},
+            notes=f"{len(notes)} vocal notes",
+        )
+
+    def _from_mix(self, audio, sample_rate, ctx):
         hop = self.hop_length
         hop_s = float(ctx.get("hop_s", hop / sample_rate))
         duration = float(ctx.get("duration_s", len(audio) / sample_rate))

musicstate-generator/src/musicstate/analyzers/stem_separation.py

@@ -0,0 +1,84 @@
+"""L2 — separate the mix into stems once, and hand the waveforms downstream.
+
+The enabler for every per-stem field: melody (vocals pitch), stems (per-bar dB
+levels), and — later — accents and chords. demucs is heavy, so the separated mono
+waveforms are cached to disk keyed by the source file, and reused on the next run.
+The waveforms are published in ctx["stems_audio"]; nothing here writes the map.
+"""
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+
+import numpy as np
+
+from ..config import DEMUCS_MODEL, SAMPLE_RATE, stems_cache_dir
+from .base import Analyzer, AnalyzerResult
+
+log = logging.getLogger("musicstate.stem_separation")
+
+_NAMES = ["drums", "bass", "other", "vocals"]
+
+
+class StemSeparationAnalyzer(Analyzer):
+    name = "demucs-separate"
+    level = "L2"
+
+    def available(self) -> tuple[bool, str]:
+        try:
+            import demucs  # noqa: F401
+            import torch  # noqa: F401
+        except Exception as exc:  # noqa: BLE001
+            return False, f"demucs/torch missing ({exc.__class__.__name__})"
+        return True, ""
+
+    def _cache_key(self, audio, ctx) -> str:
+        src = ctx.get("source_path")
+        if src:
+            return os.path.splitext(os.path.basename(src))[0]
+        return hashlib.sha1(np.asarray(audio, dtype=np.float32).tobytes()).hexdigest()[:16]
+
+    def analyze(self, audio, sample_rate, ctx):
+        cache = os.path.join(stems_cache_dir(), self._cache_key(audio, ctx))
+        paths = {n: os.path.join(cache, f"{n}.npy") for n in _NAMES}
+
+        if all(os.path.exists(p) for p in paths.values()):
+            stems = {n: np.load(p) for n, p in paths.items()}
+            log.debug("loaded cached stems from %s", cache)
+        else:
+            stems = self._separate(audio, sample_rate)
+            os.makedirs(cache, exist_ok=True)
+            for n, arr in stems.items():
+                np.save(paths[n], arr.astype(np.float32))
+            log.debug("separated and cached stems to %s", cache)
+
+        return AnalyzerResult(
+            status="ok", patch={},
+            ctx={"stems_audio": stems, "stems_sr": SAMPLE_RATE, "stems_names": _NAMES},
+            notes=f"{len(stems)} stems at {SAMPLE_RATE} Hz",
+        )
+
+    def _separate(self, audio, sample_rate):
+        import librosa
+        import torch
+        from demucs.apply import apply_model
+        from demucs.pretrained import get_model
+
+        model = get_model(DEMUCS_MODEL)
+        model.cpu().eval()
+        msr = model.samplerate
+        resampled = librosa.resample(audio, orig_sr=sample_rate, target_sr=msr)
+        wav = torch.from_numpy(np.stack([resampled, resampled])).float()
+        ref = wav.mean(0)
+        wav = (wav - ref.mean()) / (ref.std() + 1e-8)
+        with torch.no_grad():
+            sources = apply_model(model, wav[None], device="cpu", progress=False)[0]
+        sources = sources * ref.std() + ref.mean()
+        out = {}
+        for i, name in enumerate(model.sources):
+            if name not in _NAMES:
+                continue
+            mono = sources[i].mean(0).numpy()
+            out[name] = librosa.resample(mono, orig_sr=msr, target_sr=SAMPLE_RATE).astype(np.float32)
+        return out

musicstate-generator/src/musicstate/analyzers/stems.py

@@ -1,8 +1,10 @@
-"""L2 observation — Demucs source separation → per-stem presence timeline.
+"""L2 observation — per-stem loudness, in dB against the mix, one value per bar.
 
-Adds a real vocal/instrument-presence signal. Observation tier: it enriches, never
-contradicts, the L2 interface fields. Heavy (downloads htdemucs, runs a net), so it
-belongs to the deep set only.
+Consumes the separated waveforms from StemSeparationAnalyzer (ctx["stems_audio"]);
+this analyzer only measures. Each stem's RMS over a bar is expressed in dB relative
+to the mix's whole-song RMS, so summed as energy the stems track the mix bar by
+bar — which is the invariant a separator preserves and the scorer checks. (The old
+field normalised each stem by its own maximum, which cannot sum to a mix.)
 """
 from __future__ import annotations
 
@@ -20,62 +22,43 @@ class StemsAnalyzer(Analyzer):
     name = "htdemucs"
     level = "L2"
 
-    def available(self) -> tuple[bool, str]:
-        try:
-            import demucs  # noqa: F401
-            import torch  # noqa: F401
-        except Exception as exc:  # noqa: BLE001
-            return False, f"demucs/torch missing ({exc.__class__.__name__})"
-        return True, ""
-
     def analyze(self, audio, sample_rate, ctx):
-        import librosa
-        import torch
-        from demucs.apply import apply_model
-        from demucs.pretrained import get_model
-
-        model = get_model(DEMUCS_MODEL)
-        model.cpu().eval()
-        model_sr = model.samplerate
-        log.debug("separating with %s at %dHz", DEMUCS_MODEL, model_sr)
-
-        resampled = librosa.resample(audio, orig_sr=sample_rate, target_sr=model_sr)
-        wav = torch.from_numpy(np.stack([resampled, resampled])).float()  # fake stereo
-        ref = wav.mean(0)
-        wav = (wav - ref.mean()) / (ref.std() + 1e-8)
-        with torch.no_grad():
-            sources = apply_model(model, wav[None], device="cpu", progress=False)[0]
-        sources = sources * ref.std() + ref.mean()
-
-        names = list(model.sources)
-        hop = int(0.1 * model_sr)
-        levels = {}
-        for i, name in enumerate(names):
-            mono = sources[i].mean(0).numpy()
-            levels[name] = librosa.feature.rms(y=mono, hop_length=hop)[0]
-        t_axis = librosa.frames_to_time(np.arange(len(next(iter(levels.values())))),
-                                        sr=model_sr, hop_length=hop)
-
-        def presence(rms):
-            top = np.percentile(rms, 99) or 1.0
-            return np.clip(rms / top, 0, 1)
-
-        pres = {name: presence(rms) for name, rms in levels.items()}
-        downbeats = ctx.get("downbeats") or []
-        per_downbeat = []
-        for t in downbeats:
-            j = int(np.clip(np.searchsorted(t_axis, t), 0, len(t_axis) - 1))
-            per_downbeat.append({"t": round(float(t), 3),
-                                 **{name: round(float(pres[name][j]), 3) for name in names}})
-
-        vocal_fraction = None
-        if "vocals" in pres:
-            vocal_fraction = round(float((pres["vocals"] > 0.15).mean()), 3)
+        stems = ctx.get("stems_audio") or {}
+        downs = ctx.get("downbeats") or []
+        if not stems or len(downs) < 8:
+            return AnalyzerResult(status="not_computed", patch={},
+                                  notes="no separated stems or too few bars")
+
+        names = list(stems.keys())
+        audio = np.asarray(audio, dtype=np.float64)
+        mix_ref = float(np.sqrt(np.mean(audio ** 2))) or 1e-6
+        bar = float(np.median(np.diff(downs))) if len(downs) > 1 else 2.0
+
+        def bar_rms(y, t0, t1):
+            a, b = int(t0 * sample_rate), int(min(t1, len(y) / sample_rate) * sample_rate)
+            seg = np.asarray(y[a:b], dtype=np.float64)
+            return float(np.sqrt(np.mean(seg ** 2))) if len(seg) else 1e-9
+
+        sources = {n: [] for n in names}
+        for i, t in enumerate(downs):
+            t1 = downs[i + 1] if i + 1 < len(downs) else t + bar
+            for n in names:
+                sources[n].append(round(20.0 * float(np.log10(bar_rms(stems[n], t, t1) / mix_ref + 1e-9)), 2))
+
+        arrs = {n: np.array(sources[n]) for n in names}
+        levels = {n: {"present": bool(arrs[n].max() > -50.0)} for n in names}
+        vocal_fraction = (round(float((arrs["vocals"] > arrs["vocals"].max() - 20.0).mean()), 3)
+                          if "vocals" in arrs else None)
 
         return AnalyzerResult(
             status="ok",
-            patch={"stems": {"model": DEMUCS_MODEL, "names": names, "rate": "per_downbeat",
-                             "per_downbeat": per_downbeat, "vocal_present_fraction": vocal_fraction}},
-            confidence={"stems": 0.6},
-            notes="per-stem presence vs own 99th pct; observation tier",
+            patch={"stems": {
+                "model": DEMUCS_MODEL, "names": names, "rate": "per_downbeat",
+                "comparable": True,
+                "unit": "dB of the stem's RMS over one bar, against the mix's whole-song RMS",
+                "sources": sources, "at": [round(float(t), 3) for t in downs],
+                "levels": levels, "vocal_present_fraction": vocal_fraction,
+            }},
+            confidence={"stems": 0.7},
+            notes="per-bar dB levels vs the mix (comparable)",
         )

musicstate-generator/src/musicstate/config.py

@@ -70,6 +70,15 @@ def _sibling_env_python(env_name: str, override_var: str) -> str:
     return "python"
 
 
+_STEMS_CACHE = os.path.normpath(
+    os.path.join(os.path.dirname(__file__), "..", "..", "..", "work", "stems_cache"))
+
+
+def stems_cache_dir() -> str:
+    """Where separated stem waveforms are cached, so demucs runs once per song."""
+    return os.environ.get("LIMELIGHT_STEMS_CACHE", _STEMS_CACHE)
+
+
 def essentia_python() -> str:
     return _sibling_env_python("limelight-ess", "LIMELIGHT_ESS_PY")
 

musicstate-generator/src/musicstate/pipeline.py

@@ -28,6 +28,7 @@
     NoteTranscriptionAnalyzer,
     PumpAnalyzer,
     SemanticAnalyzer,
+    StemSeparationAnalyzer,
     StemsAnalyzer,
     StructureAnalyzer,
 )
@@ -52,6 +53,7 @@ def deep_analyzers() -> list[Analyzer]:
         GridRefineAnalyzer(),    # rigid clock phase-locked to the kick
         BarPhaseAnalyzer(),      # kick-band downbeat phase (allin1 as strong prior)
         MomentDeriveAnalyzer(),  # candidate drops/quiets from labels + energy
+        StemSeparationAnalyzer(),  # demucs once (cached) -> waveforms for the per-stem fields
         AccentsAnalyzer(),       # discrete onsets on the (allin1) beat grid
         ChordsAnalyzer(),        # per-bar chords on the (allin1) beat grid
         MelodyAnalyzer(),        # pyin melody contour

musicstate-generator/src/musicstate/port.py

@@ -167,8 +167,14 @@ def to_map(state: dict, vec_filename: str | None = None,
     period = round(60.0 / bpm, 5) if bpm else None
     phase = beats[0] if beats else 0.0
     bar_phase = beats.index(downbeats[0]) if (downbeats and downbeats[0] in beats) else 0
+    ts = str(meta.get("time_signature") or "4/4")
+    try:
+        beats_per_bar = int(ts.split("/")[0])
+    except (ValueError, IndexError):
+        beats_per_bar = 4
     grid = {
         "period": period, "phase": phase, "bpm": bpm, "bar_phase": bar_phase,
+        "beats_per_bar": beats_per_bar,
         "locked": False, "how": "allin1 beat tracking",
         "note": "beats start at the first tracked beat, not at t=0",
     }
@@ -240,19 +246,27 @@ def to_map(state: dict, vec_filename: str | None = None,
         spans.append({"kind": "build", "from": frm, "to": to, "rise": "steady",
                       "bars": bars, "how": "derived: build moment to the next drop moment"})
 
-    # ---- stems: list-of-dicts -> per-stem arrays + explicit guitar/piano zeros ----
+    # ---- stems: per-bar levels, passed through ----
     st = state.get("stems") or {}
-    pd = st.get("per_downbeat") or []
-    names = st.get("names") or []
-    at = [r["t"] for r in pd]
-    sources = {nm: [r.get(nm) for r in pd] for nm in names}
-    for extra in ("guitar", "piano"):
-        sources.setdefault(extra, [0.0] * len(pd))
-    stems = {
-        "model": st.get("model"), "rate": "per_downbeat", "sources": sources, "at": at,
-        "vocal_present_fraction": st.get("vocal_present_fraction"),
-        "note": "Four-stem htdemucs mapped onto the six canonical names.",
-    } if pd else None
+    stems = None
+    if st.get("sources"):
+        comparable = st.get("comparable") is True
+        sources = {nm: list(vals) for nm, vals in st["sources"].items()}
+        n = len(st.get("at") or next(iter(sources.values()), []))
+        # htdemucs 4-stem does not separate guitar/piano (they fold into 'other'); mark
+        # them absent, NOT zero -- a 0 dB level would add spurious energy under the
+        # comparable-sum check that ev_stems runs.
+        fill = -120.0 if comparable else 0.0
+        for extra in ("guitar", "piano"):
+            sources.setdefault(extra, [fill] * n)
+        stems = {
+            "model": st.get("model"), "rate": st.get("rate", "per_downbeat"),
+            "comparable": comparable, "unit": st.get("unit"),
+            "sources": sources, "at": st.get("at"),
+            "levels": st.get("levels"),
+            "vocal_present_fraction": st.get("vocal_present_fraction"),
+            "note": "Four-stem htdemucs; guitar/piano fold into 'other' and are marked absent.",
+        }
 
     # ---- observation tier ----
     sem = state.get("semantic") or {}

musicstate-generator/tests/test_port.py

@@ -29,8 +29,11 @@
         {"t": 26.0, "type": "wobble", "conf": 0.9},  # not one of the six -> dropped
     ],
     "stems": {"model": "htdemucs", "names": ["drums", "vocals"], "rate": "per_downbeat",
-              "per_downbeat": [{"t": 0.5, "drums": 0.2, "vocals": 0.1},
-                               {"t": 2.0, "drums": 0.4, "vocals": 0.6}],
+              "comparable": True,
+              "unit": "dB of the stem's RMS over one bar, against the mix's whole-song RMS",
+              "sources": {"drums": [-24.0, -22.0], "vocals": [-30.0, -28.0]},
+              "at": [0.5, 2.0],
+              "levels": {"drums": {"present": True}, "vocals": {"present": True}},
               "vocal_present_fraction": 0.5},
     "chords": {"rate": "per_bar", "how": "chroma templates",
                "events": [{"at": 0.5, "chord": "Am", "confidence": 0.8}]},
@@ -89,10 +92,13 @@ def test_confidence_is_mean_of_fields():
     assert _m()["confidence"] == 0.7  # mean(0.8, 0.6)
 
 
-def test_stems_reshape_with_guitar_piano_zeros():
+def test_stems_reshape_comparable_db_with_guitar_piano_absent():
     s = _m()["stems"]
+    assert s["comparable"] is True
     assert set(s["sources"]) == {"drums", "vocals", "guitar", "piano"}
-    assert s["sources"]["guitar"] == [0.0, 0.0] and s["sources"]["piano"] == [0.0, 0.0]
+    # dB levels pass through; guitar/piano are marked absent at a floor, not 0 dB
+    assert s["sources"]["drums"] == [-24.0, -22.0]
+    assert s["sources"]["guitar"] == [-120.0, -120.0] and s["sources"]["piano"] == [-120.0, -120.0]
     assert s["at"] == [0.5, 2.0]