diff --git a/README.md b/README.md index d6773e8..19fcca6 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ An API and command line interface (CLI) are provided to allow users to download * OpenMidnight-1.1: https://huggingface.co/medarc/OpenMidnight-1.1 * OpenMidnight-1.1-SPECTRA: https://huggingface.co/medarc/OpenMidnight-1.1-SPECTRA * H-optimus-0-SPECTRA: https://huggingface.co/medarc/H-optimus-0-SPECTRA +* METTLE 1.5: https://huggingface.co/slideflow-labs/mettle-1.5 * Virchow-SPECTRA: https://huggingface.co/medarc/Virchow-SPECTRA * Midnight-12k-SPECTRA: https://huggingface.co/medarc/Midnight-12k-SPECTRA * GenBio-PathFM: https://huggingface.co/genbio-ai/genbio-pathfm diff --git a/docs/index.md b/docs/index.md index 6a92df5..c0db974 100644 --- a/docs/index.md +++ b/docs/index.md @@ -119,6 +119,7 @@ pip install . # install the package |HIBOU-B | hiboub | ViT-B/14 | 86M | DINOv2 | |x | |HIBOU-L | hiboul | ViT-L/14 | 307M | DINOv2 | |x | |H−OPTIMUS−0 | hoptimus0 | ViT-G/14 | 1.1B | DINOv2 | |x | +|METTLE 1.5 | mettle15 | ViT-G/14 | 1.1B | Fine-tuning (base: H−OPTIMUS−0) | |x | |H0-mini | h0mini | ViT-B/14 | 86M | DINOv2 (teacher: H−OPTIMUS−0) | |x | |H−OPTIMUS−1 | hoptimus1 | ViT-G/14 | 1.1B | DINOv2 | |x | |KAIKO-S/8 | kaiko_vits8 | ViT-S/8 | 22M | DINO | |x | diff --git a/src/thunder/benchmark.py b/src/thunder/benchmark.py index 448eb8a..311133f 100644 --- a/src/thunder/benchmark.py +++ b/src/thunder/benchmark.py @@ -21,7 +21,7 @@ def benchmark( where options are: - dataset: *bach*, *bracs*, *break_his*, *ccrcc*, *crc*, *esca*, *mhist*, *ocelot*, *pannuke*, *patch_camelyon*, *segpath_epithelial*, *segpath_lymphocytes*, *tcga_crc_msi*, *tcga_tils*, *tcga_uniform*, *wilds* - - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *openmidnight11*, *openmidnight11_spectra*, *hoptimus0_spectra*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14* + - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *openmidnight11*, *openmidnight11_spectra*, *hoptimus0_spectra*, *mettle15*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14* - task: *adversarial_attack*, *alignment_scoring*, *image_retrieval*, *knn*, *linear_probing*, *pre_computing_embeddings*, *segmentation*, *simple_shot*, *transformation_invariance*, *zero_shot_vlm* - loading_mode: *online_loading*, *image_pre_loading*, *embedding_pre_loading* diff --git a/src/thunder/config/pretrained_model/mettle15.yaml b/src/thunder/config/pretrained_model/mettle15.yaml new file mode 100644 index 0000000..f29b0a7 --- /dev/null +++ b/src/thunder/config/pretrained_model/mettle15.yaml @@ -0,0 +1,6 @@ +model_name: mettle15 +type: safetensors +vlm: false +emb_dim: 1536 +ckpt_path: ${oc.env:THUNDER_BASE_DATA_FOLDER}/pretrained_ckpts/mettle1.5/ +hf_tag: hf-hub:slideflow-labs/mettle-1.5 diff --git a/src/thunder/models/adapters.py b/src/thunder/models/adapters.py index 3a910af..6312e42 100644 --- a/src/thunder/models/adapters.py +++ b/src/thunder/models/adapters.py @@ -39,6 +39,9 @@ def get_model_lora_names( elif model_name == "conch": block_name = "visual.trunk.blocks" qkv_name = "attn.qkv" + elif model_name == "mettle15": + block_name = "backbone.blocks" + qkv_name = "attn.qkv" else: raise NotImplementedError( f"LoRA not implemented for {model_name} and {model_type} model type." diff --git a/src/thunder/models/download.py b/src/thunder/models/download.py index b343f0e..9981f5a 100644 --- a/src/thunder/models/download.py +++ b/src/thunder/models/download.py @@ -34,6 +34,10 @@ "medarc/H-optimus-0-SPECTRA", "model.safetensors", ), # H-optimus-0-SPECTRA (https://huggingface.co/medarc/H-optimus-0-SPECTRA) + "mettle15": ( + "slideflow-labs/mettle-1.5", + "model.safetensors", + ), # METTLE 1.5 (https://huggingface.co/slideflow-labs/mettle-1.5) "hoptimus0": ( "bioptimus/H-optimus-0", "pytorch_model.bin", @@ -180,6 +184,7 @@ def download_models(models: Union[List[str], str]) -> None: * virchow2 * hoptimus0 * hoptimus0_spectra + * mettle15 * h0mini * hoptimus1 * genbio-pathfm @@ -296,6 +301,13 @@ def download_model(model: str) -> None: "virchowspectra", ]: extra_files = ["config.json"] + elif local_dir_tag == "mettle1.5": + extra_files = [ + "config.json", + "preprocessor_config.json", + "configuration_mettle.py", + "modeling_mettle.py", + ] elif local_dir_tag == "titan": extra_files = [ "text_transformer.py", diff --git a/src/thunder/models/pretrained_models.py b/src/thunder/models/pretrained_models.py index d25d54e..b25acb2 100644 --- a/src/thunder/models/pretrained_models.py +++ b/src/thunder/models/pretrained_models.py @@ -130,6 +130,8 @@ def get_model(model_cfg: dict, device: str): model, transform = get_midnight(model_cfg.ckpt_path) elif model_cfg.model_name == "genbio-pathfm": model, transform = get_genbio_pathfm(model_cfg.ckpt_path) + elif model_cfg.model_name == "mettle15": + model, transform = get_mettle(model_cfg.ckpt_path) else: model, transform, tokenizer = get_from_safetensors( model_cfg.ckpt_path, use_fast="dinov3" in model_cfg.model_name @@ -348,6 +350,17 @@ def extract_embedding(src, pretrained_model, task_type): else: return patch_tokens + elif model_cfg.model_name == "mettle15": + + def extract_embedding(src, pretrained_model, task_type="linear_probing"): + out = pretrained_model(src) + if task_type != "segmentation": + # Refined CLS embedding (the model's default feature view). + return out.pooler_output + else: + # Spatial patch tokens, ignoring CLS and the 4 register tokens. + return out.last_hidden_state[:, 5:] + elif model_cfg.model_name in ["midnight", "midnight12k_spectra"]: def extract_embedding(src, pretrained_model, task_type): @@ -406,6 +419,7 @@ def get_model_from_name(model_name: str, device: str): * virchow2 * hoptimus0 * hoptimus0_spectra + * mettle15 * h0mini * hoptimus1 * genbio-pathfm @@ -819,3 +833,31 @@ def get_genbio_pathfm(ckpt_path: str): return model, transform + +def get_mettle(ckpt_path: str): + """ + Adapted from: + - https://huggingface.co/slideflow-labs/mettle-1.5 + + :param ckpt_path: path to the folder with the stored checkpoint and model code. + """ + from torchvision import transforms + + # Model + model = AutoModel.from_pretrained(ckpt_path, trust_remote_code=True) + + # Transform (as specified in the model's preprocessor_config.json) + transform = transforms.Compose( + [ + transforms.Resize( + (224, 224), interpolation=transforms.InterpolationMode.BICUBIC + ), + transforms.ToTensor(), + transforms.Normalize( + mean=(0.707223, 0.578729, 0.703617), + std=(0.211883, 0.230117, 0.177517), + ), + ] + ) + + return model, transform diff --git a/src/thunder/utils/constants.py b/src/thunder/utils/constants.py index 849ae27..07869fc 100644 --- a/src/thunder/utils/constants.py +++ b/src/thunder/utils/constants.py @@ -8,6 +8,7 @@ class ModelConstants(Enum): "hiboul", "hoptimus0", "hoptimus0_spectra", + "mettle15", "h0mini", "hoptimus1", "genbio-pathfm",