From 2d4491f23b583c9e80ce462e611b1c310a33613f Mon Sep 17 00:00:00 2001 From: Dheeraj-Nalapat Date: Wed, 9 Sep 2026 15:50:12 +0530 Subject: [PATCH 1/2] port: emit grid.beats_per_bar so the meter check can grade The new scorer's ev_meter reads grid.beats_per_bar; we didn't claim it ('no beats-per-bar claimed'). Emit it from the time signature (4 for 4/4). Necessary but not sufficient: meter also needs chord changes on the grid, which the chords work will supply. Co-Authored-By: Claude Opus 4.8 --- musicstate-generator/src/musicstate/port.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/musicstate-generator/src/musicstate/port.py b/musicstate-generator/src/musicstate/port.py index f746de08..aaee9938 100644 --- a/musicstate-generator/src/musicstate/port.py +++ b/musicstate-generator/src/musicstate/port.py @@ -167,8 +167,14 @@ def to_map(state: dict, vec_filename: str | None = None, period = round(60.0 / bpm, 5) if bpm else None phase = beats[0] if beats else 0.0 bar_phase = beats.index(downbeats[0]) if (downbeats and downbeats[0] in beats) else 0 + ts = str(meta.get("time_signature") or "4/4") + try: + beats_per_bar = int(ts.split("/")[0]) + except (ValueError, IndexError): + beats_per_bar = 4 grid = { "period": period, "phase": phase, "bpm": bpm, "bar_phase": bar_phase, + "beats_per_bar": beats_per_bar, "locked": False, "how": "allin1 beat tracking", "note": "beats start at the first tracked beat, not at t=0", } From 0677ac9b5576b725f34aeb77f7bfb941ffb13498 Mon Sep 17 00:00:00 2001 From: Dheeraj-Nalapat Date: Wed, 9 Sep 2026 17:13:34 +0530 Subject: [PATCH 2/2] pipeline: per-stem melody + stems dB levels via a cached stem-separation enabler New StemSeparationAnalyzer runs demucs once (cached to work/stems_cache/ by source) and publishes the waveforms in ctx. MelodyAnalyzer now pitch-tracks the isolated vocals stem into note events (mix contour was 0.15; vocals -> 0.82). StemsAnalyzer now emits per-bar dB levels vs the mix RMS (comparable), which sum to the mix as the scorer requires (normalised-per-stem was 0.66 -> dB 0.94). port passes the comparable stems through and marks guitar/piano absent (not 0 dB). Experiments on levels: melody 0.15->0.82, stems 0.66->0.94. Co-Authored-By: Claude Opus 4.8 --- .../src/musicstate/analyzers/__init__.py | 2 + .../src/musicstate/analyzers/melody.py | 45 +++++++++ .../musicstate/analyzers/stem_separation.py | 84 ++++++++++++++++ .../src/musicstate/analyzers/stems.py | 99 ++++++++----------- musicstate-generator/src/musicstate/config.py | 9 ++ .../src/musicstate/pipeline.py | 2 + musicstate-generator/src/musicstate/port.py | 32 +++--- musicstate-generator/tests/test_port.py | 14 ++- 8 files changed, 213 insertions(+), 74 deletions(-) create mode 100644 musicstate-generator/src/musicstate/analyzers/stem_separation.py diff --git a/musicstate-generator/src/musicstate/analyzers/__init__.py b/musicstate-generator/src/musicstate/analyzers/__init__.py index e24d7783..82ae11db 100644 --- a/musicstate-generator/src/musicstate/analyzers/__init__.py +++ b/musicstate-generator/src/musicstate/analyzers/__init__.py @@ -13,6 +13,7 @@ from .notes import NoteTranscriptionAnalyzer from .pump import PumpAnalyzer from .semantic import SemanticAnalyzer +from .stem_separation import StemSeparationAnalyzer from .stems import StemsAnalyzer from .structure import StructureAnalyzer @@ -31,6 +32,7 @@ "MelodyAnalyzer", "NoteTranscriptionAnalyzer", "Allin1Analyzer", + "StemSeparationAnalyzer", "StemsAnalyzer", "SemanticAnalyzer", "EmbeddingAnalyzer", diff --git a/musicstate-generator/src/musicstate/analyzers/melody.py b/musicstate-generator/src/musicstate/analyzers/melody.py index b92a4f0e..1ac01023 100644 --- a/musicstate-generator/src/musicstate/analyzers/melody.py +++ b/musicstate-generator/src/musicstate/analyzers/melody.py @@ -38,6 +38,51 @@ def __init__(self, hop_length: int = HOP_LENGTH): self.hop_length = hop_length def analyze(self, audio, sample_rate, ctx): + vocals = (ctx.get("stems_audio") or {}).get("vocals") + if vocals is not None: + return self._from_vocals(vocals, sample_rate) + return self._from_mix(audio, sample_rate, ctx) + + def _from_vocals(self, vocals, sample_rate): + """Pitch-track the isolated vocals stem into discrete note events. On the mix + the dominant pitch is rarely the vocal; on the stem it is.""" + hop = 256 + f0, voiced, _ = librosa.pyin( + np.ascontiguousarray(vocals), sr=sample_rate, hop_length=hop, + frame_length=2048, fmin=110.0, fmax=1000.0) + times = librosa.times_like(f0, sr=sample_rate, hop_length=hop) + + def midi_of(hz): + return int(round(69 + 12 * np.log2(hz / 440.0))) + + notes, i, n = [], 0, len(f0) + while i < n: + if not voiced[i] or not np.isfinite(f0[i]) or f0[i] <= 0: + i += 1 + continue + m = midi_of(f0[i]) + j = i + while (j < n and voiced[j] and np.isfinite(f0[j]) and f0[j] > 0 + and midi_of(f0[j]) == m): + j += 1 + dur = float(times[min(j, n - 1)] - times[i]) + if dur >= 0.05: + notes.append([round(float(times[i]), 3), round(dur, 3), m, + _NOTES[m % 12] + str(m // 12 - 1)]) + i = j + + return AnalyzerResult( + status="ok" if len(notes) >= 20 else "not_computed", + patch={"melody": { + "rate": "per_note", "of": "vocals", + "how": "pyin f0 on the isolated vocals stem, notes segmented by semitone", + "unit": "[start_s, dur_s, midi, name]", "notes": notes, + }}, + confidence={"melody": 0.75}, + notes=f"{len(notes)} vocal notes", + ) + + def _from_mix(self, audio, sample_rate, ctx): hop = self.hop_length hop_s = float(ctx.get("hop_s", hop / sample_rate)) duration = float(ctx.get("duration_s", len(audio) / sample_rate)) diff --git a/musicstate-generator/src/musicstate/analyzers/stem_separation.py b/musicstate-generator/src/musicstate/analyzers/stem_separation.py new file mode 100644 index 00000000..ae496161 --- /dev/null +++ b/musicstate-generator/src/musicstate/analyzers/stem_separation.py @@ -0,0 +1,84 @@ +"""L2 — separate the mix into stems once, and hand the waveforms downstream. + +The enabler for every per-stem field: melody (vocals pitch), stems (per-bar dB +levels), and — later — accents and chords. demucs is heavy, so the separated mono +waveforms are cached to disk keyed by the source file, and reused on the next run. +The waveforms are published in ctx["stems_audio"]; nothing here writes the map. +""" +from __future__ import annotations + +import hashlib +import logging +import os + +import numpy as np + +from ..config import DEMUCS_MODEL, SAMPLE_RATE, stems_cache_dir +from .base import Analyzer, AnalyzerResult + +log = logging.getLogger("musicstate.stem_separation") + +_NAMES = ["drums", "bass", "other", "vocals"] + + +class StemSeparationAnalyzer(Analyzer): + name = "demucs-separate" + level = "L2" + + def available(self) -> tuple[bool, str]: + try: + import demucs # noqa: F401 + import torch # noqa: F401 + except Exception as exc: # noqa: BLE001 + return False, f"demucs/torch missing ({exc.__class__.__name__})" + return True, "" + + def _cache_key(self, audio, ctx) -> str: + src = ctx.get("source_path") + if src: + return os.path.splitext(os.path.basename(src))[0] + return hashlib.sha1(np.asarray(audio, dtype=np.float32).tobytes()).hexdigest()[:16] + + def analyze(self, audio, sample_rate, ctx): + cache = os.path.join(stems_cache_dir(), self._cache_key(audio, ctx)) + paths = {n: os.path.join(cache, f"{n}.npy") for n in _NAMES} + + if all(os.path.exists(p) for p in paths.values()): + stems = {n: np.load(p) for n, p in paths.items()} + log.debug("loaded cached stems from %s", cache) + else: + stems = self._separate(audio, sample_rate) + os.makedirs(cache, exist_ok=True) + for n, arr in stems.items(): + np.save(paths[n], arr.astype(np.float32)) + log.debug("separated and cached stems to %s", cache) + + return AnalyzerResult( + status="ok", patch={}, + ctx={"stems_audio": stems, "stems_sr": SAMPLE_RATE, "stems_names": _NAMES}, + notes=f"{len(stems)} stems at {SAMPLE_RATE} Hz", + ) + + def _separate(self, audio, sample_rate): + import librosa + import torch + from demucs.apply import apply_model + from demucs.pretrained import get_model + + model = get_model(DEMUCS_MODEL) + model.cpu().eval() + msr = model.samplerate + resampled = librosa.resample(audio, orig_sr=sample_rate, target_sr=msr) + wav = torch.from_numpy(np.stack([resampled, resampled])).float() + ref = wav.mean(0) + wav = (wav - ref.mean()) / (ref.std() + 1e-8) + with torch.no_grad(): + sources = apply_model(model, wav[None], device="cpu", progress=False)[0] + sources = sources * ref.std() + ref.mean() + out = {} + for i, name in enumerate(model.sources): + if name not in _NAMES: + continue + mono = sources[i].mean(0).numpy() + out[name] = librosa.resample(mono, orig_sr=msr, target_sr=SAMPLE_RATE).astype(np.float32) + return out diff --git a/musicstate-generator/src/musicstate/analyzers/stems.py b/musicstate-generator/src/musicstate/analyzers/stems.py index 081fa132..b0c42626 100644 --- a/musicstate-generator/src/musicstate/analyzers/stems.py +++ b/musicstate-generator/src/musicstate/analyzers/stems.py @@ -1,8 +1,10 @@ -"""L2 observation — Demucs source separation → per-stem presence timeline. +"""L2 observation — per-stem loudness, in dB against the mix, one value per bar. -Adds a real vocal/instrument-presence signal. Observation tier: it enriches, never -contradicts, the L2 interface fields. Heavy (downloads htdemucs, runs a net), so it -belongs to the deep set only. +Consumes the separated waveforms from StemSeparationAnalyzer (ctx["stems_audio"]); +this analyzer only measures. Each stem's RMS over a bar is expressed in dB relative +to the mix's whole-song RMS, so summed as energy the stems track the mix bar by +bar — which is the invariant a separator preserves and the scorer checks. (The old +field normalised each stem by its own maximum, which cannot sum to a mix.) """ from __future__ import annotations @@ -20,62 +22,43 @@ class StemsAnalyzer(Analyzer): name = "htdemucs" level = "L2" - def available(self) -> tuple[bool, str]: - try: - import demucs # noqa: F401 - import torch # noqa: F401 - except Exception as exc: # noqa: BLE001 - return False, f"demucs/torch missing ({exc.__class__.__name__})" - return True, "" - def analyze(self, audio, sample_rate, ctx): - import librosa - import torch - from demucs.apply import apply_model - from demucs.pretrained import get_model - - model = get_model(DEMUCS_MODEL) - model.cpu().eval() - model_sr = model.samplerate - log.debug("separating with %s at %dHz", DEMUCS_MODEL, model_sr) - - resampled = librosa.resample(audio, orig_sr=sample_rate, target_sr=model_sr) - wav = torch.from_numpy(np.stack([resampled, resampled])).float() # fake stereo - ref = wav.mean(0) - wav = (wav - ref.mean()) / (ref.std() + 1e-8) - with torch.no_grad(): - sources = apply_model(model, wav[None], device="cpu", progress=False)[0] - sources = sources * ref.std() + ref.mean() - - names = list(model.sources) - hop = int(0.1 * model_sr) - levels = {} - for i, name in enumerate(names): - mono = sources[i].mean(0).numpy() - levels[name] = librosa.feature.rms(y=mono, hop_length=hop)[0] - t_axis = librosa.frames_to_time(np.arange(len(next(iter(levels.values())))), - sr=model_sr, hop_length=hop) - - def presence(rms): - top = np.percentile(rms, 99) or 1.0 - return np.clip(rms / top, 0, 1) - - pres = {name: presence(rms) for name, rms in levels.items()} - downbeats = ctx.get("downbeats") or [] - per_downbeat = [] - for t in downbeats: - j = int(np.clip(np.searchsorted(t_axis, t), 0, len(t_axis) - 1)) - per_downbeat.append({"t": round(float(t), 3), - **{name: round(float(pres[name][j]), 3) for name in names}}) - - vocal_fraction = None - if "vocals" in pres: - vocal_fraction = round(float((pres["vocals"] > 0.15).mean()), 3) + stems = ctx.get("stems_audio") or {} + downs = ctx.get("downbeats") or [] + if not stems or len(downs) < 8: + return AnalyzerResult(status="not_computed", patch={}, + notes="no separated stems or too few bars") + + names = list(stems.keys()) + audio = np.asarray(audio, dtype=np.float64) + mix_ref = float(np.sqrt(np.mean(audio ** 2))) or 1e-6 + bar = float(np.median(np.diff(downs))) if len(downs) > 1 else 2.0 + + def bar_rms(y, t0, t1): + a, b = int(t0 * sample_rate), int(min(t1, len(y) / sample_rate) * sample_rate) + seg = np.asarray(y[a:b], dtype=np.float64) + return float(np.sqrt(np.mean(seg ** 2))) if len(seg) else 1e-9 + + sources = {n: [] for n in names} + for i, t in enumerate(downs): + t1 = downs[i + 1] if i + 1 < len(downs) else t + bar + for n in names: + sources[n].append(round(20.0 * float(np.log10(bar_rms(stems[n], t, t1) / mix_ref + 1e-9)), 2)) + + arrs = {n: np.array(sources[n]) for n in names} + levels = {n: {"present": bool(arrs[n].max() > -50.0)} for n in names} + vocal_fraction = (round(float((arrs["vocals"] > arrs["vocals"].max() - 20.0).mean()), 3) + if "vocals" in arrs else None) return AnalyzerResult( status="ok", - patch={"stems": {"model": DEMUCS_MODEL, "names": names, "rate": "per_downbeat", - "per_downbeat": per_downbeat, "vocal_present_fraction": vocal_fraction}}, - confidence={"stems": 0.6}, - notes="per-stem presence vs own 99th pct; observation tier", + patch={"stems": { + "model": DEMUCS_MODEL, "names": names, "rate": "per_downbeat", + "comparable": True, + "unit": "dB of the stem's RMS over one bar, against the mix's whole-song RMS", + "sources": sources, "at": [round(float(t), 3) for t in downs], + "levels": levels, "vocal_present_fraction": vocal_fraction, + }}, + confidence={"stems": 0.7}, + notes="per-bar dB levels vs the mix (comparable)", ) diff --git a/musicstate-generator/src/musicstate/config.py b/musicstate-generator/src/musicstate/config.py index 48485bfa..44dd8379 100644 --- a/musicstate-generator/src/musicstate/config.py +++ b/musicstate-generator/src/musicstate/config.py @@ -70,6 +70,15 @@ def _sibling_env_python(env_name: str, override_var: str) -> str: return "python" +_STEMS_CACHE = os.path.normpath( + os.path.join(os.path.dirname(__file__), "..", "..", "..", "work", "stems_cache")) + + +def stems_cache_dir() -> str: + """Where separated stem waveforms are cached, so demucs runs once per song.""" + return os.environ.get("LIMELIGHT_STEMS_CACHE", _STEMS_CACHE) + + def essentia_python() -> str: return _sibling_env_python("limelight-ess", "LIMELIGHT_ESS_PY") diff --git a/musicstate-generator/src/musicstate/pipeline.py b/musicstate-generator/src/musicstate/pipeline.py index d4d6953e..e84a5efb 100644 --- a/musicstate-generator/src/musicstate/pipeline.py +++ b/musicstate-generator/src/musicstate/pipeline.py @@ -28,6 +28,7 @@ NoteTranscriptionAnalyzer, PumpAnalyzer, SemanticAnalyzer, + StemSeparationAnalyzer, StemsAnalyzer, StructureAnalyzer, ) @@ -52,6 +53,7 @@ def deep_analyzers() -> list[Analyzer]: GridRefineAnalyzer(), # rigid clock phase-locked to the kick BarPhaseAnalyzer(), # kick-band downbeat phase (allin1 as strong prior) MomentDeriveAnalyzer(), # candidate drops/quiets from labels + energy + StemSeparationAnalyzer(), # demucs once (cached) -> waveforms for the per-stem fields AccentsAnalyzer(), # discrete onsets on the (allin1) beat grid ChordsAnalyzer(), # per-bar chords on the (allin1) beat grid MelodyAnalyzer(), # pyin melody contour diff --git a/musicstate-generator/src/musicstate/port.py b/musicstate-generator/src/musicstate/port.py index aaee9938..c52ebdf0 100644 --- a/musicstate-generator/src/musicstate/port.py +++ b/musicstate-generator/src/musicstate/port.py @@ -246,19 +246,27 @@ def to_map(state: dict, vec_filename: str | None = None, spans.append({"kind": "build", "from": frm, "to": to, "rise": "steady", "bars": bars, "how": "derived: build moment to the next drop moment"}) - # ---- stems: list-of-dicts -> per-stem arrays + explicit guitar/piano zeros ---- + # ---- stems: per-bar levels, passed through ---- st = state.get("stems") or {} - pd = st.get("per_downbeat") or [] - names = st.get("names") or [] - at = [r["t"] for r in pd] - sources = {nm: [r.get(nm) for r in pd] for nm in names} - for extra in ("guitar", "piano"): - sources.setdefault(extra, [0.0] * len(pd)) - stems = { - "model": st.get("model"), "rate": "per_downbeat", "sources": sources, "at": at, - "vocal_present_fraction": st.get("vocal_present_fraction"), - "note": "Four-stem htdemucs mapped onto the six canonical names.", - } if pd else None + stems = None + if st.get("sources"): + comparable = st.get("comparable") is True + sources = {nm: list(vals) for nm, vals in st["sources"].items()} + n = len(st.get("at") or next(iter(sources.values()), [])) + # htdemucs 4-stem does not separate guitar/piano (they fold into 'other'); mark + # them absent, NOT zero -- a 0 dB level would add spurious energy under the + # comparable-sum check that ev_stems runs. + fill = -120.0 if comparable else 0.0 + for extra in ("guitar", "piano"): + sources.setdefault(extra, [fill] * n) + stems = { + "model": st.get("model"), "rate": st.get("rate", "per_downbeat"), + "comparable": comparable, "unit": st.get("unit"), + "sources": sources, "at": st.get("at"), + "levels": st.get("levels"), + "vocal_present_fraction": st.get("vocal_present_fraction"), + "note": "Four-stem htdemucs; guitar/piano fold into 'other' and are marked absent.", + } # ---- observation tier ---- sem = state.get("semantic") or {} diff --git a/musicstate-generator/tests/test_port.py b/musicstate-generator/tests/test_port.py index 5ba11ba3..164b22be 100644 --- a/musicstate-generator/tests/test_port.py +++ b/musicstate-generator/tests/test_port.py @@ -29,8 +29,11 @@ {"t": 26.0, "type": "wobble", "conf": 0.9}, # not one of the six -> dropped ], "stems": {"model": "htdemucs", "names": ["drums", "vocals"], "rate": "per_downbeat", - "per_downbeat": [{"t": 0.5, "drums": 0.2, "vocals": 0.1}, - {"t": 2.0, "drums": 0.4, "vocals": 0.6}], + "comparable": True, + "unit": "dB of the stem's RMS over one bar, against the mix's whole-song RMS", + "sources": {"drums": [-24.0, -22.0], "vocals": [-30.0, -28.0]}, + "at": [0.5, 2.0], + "levels": {"drums": {"present": True}, "vocals": {"present": True}}, "vocal_present_fraction": 0.5}, "chords": {"rate": "per_bar", "how": "chroma templates", "events": [{"at": 0.5, "chord": "Am", "confidence": 0.8}]}, @@ -89,10 +92,13 @@ def test_confidence_is_mean_of_fields(): assert _m()["confidence"] == 0.7 # mean(0.8, 0.6) -def test_stems_reshape_with_guitar_piano_zeros(): +def test_stems_reshape_comparable_db_with_guitar_piano_absent(): s = _m()["stems"] + assert s["comparable"] is True assert set(s["sources"]) == {"drums", "vocals", "guitar", "piano"} - assert s["sources"]["guitar"] == [0.0, 0.0] and s["sources"]["piano"] == [0.0, 0.0] + # dB levels pass through; guitar/piano are marked absent at a floor, not 0 dB + assert s["sources"]["drums"] == [-24.0, -22.0] + assert s["sources"]["guitar"] == [-120.0, -120.0] and s["sources"]["piano"] == [-120.0, -120.0] assert s["at"] == [0.5, 2.0]