[Draft] hg model integration (v0)

.coveragerc

@@ -0,0 +1,5 @@
+[report]
+include_namespace_packages = true
+
+[run]
+omit = tests/*

.github/workflows/_build_doc.yaml

@@ -64,7 +64,7 @@ jobs:
           pip install --editable native/python
       - name: Install fairseq2
         run: |
-          pip install --editable .
+          pip install --editable ".[hg]"
       - name: Generate documentation
         working-directory: doc
         run: |

.github/workflows/_build_wheel-linux.yaml

@@ -193,7 +193,7 @@ jobs:
           whl=$(ls ~/artifacts/build/wheelhouse/*.whl)
 
           pip install --no-cache-dir\
-            "fairseq2@file://$whl" "fairseq2[arrow]@file://$whl"
+            "fairseq2@file://$whl" "fairseq2[arrow,hg]@file://$whl"
       - name: Set the sanitizer variables
         if: inputs.sanitizers != 'nosan'
         env:

.github/workflows/_build_wheel-macos.yaml

@@ -127,7 +127,7 @@ jobs:
           whl=$(ls ~/artifacts/build/wheelhouse/*.whl)
 
           pip install --no-cache-dir\
-            "fairseq2@file://$whl" "fairseq2[arrow]@file://$whl"
+            "fairseq2@file://$whl" "fairseq2[arrow,hg]@file://$whl"
       - name: Run native tests
         run: |
           chmod 755 ~/artifacts/native/build/tests/run-tests

.github/workflows/_lint_py.yaml

@@ -53,7 +53,7 @@ jobs:
       - name: Install fairseq2
         id: install_fairseq2
         run: |
-          pip install --editable .
+          pip install --editable ".[hg]"
       - name: Run isort
         if: success() || (failure() && steps.install_fairseq2.outcome == 'success')
         run: |

.gitignore

@@ -21,3 +21,9 @@ __pycache__/
 
 # Other
 .DS_Store
+
+# coverage
+.coverage
+
+# mypy
+.mypy_cache/

doc/source/reference/api/fairseq2.models.hg.rst

@@ -0,0 +1,193 @@
+fairseq2.models.hg
+==================
+
+The :mod:`fairseq2.models.hg` module provides seamless integration with HuggingFace Transformers models within the fairseq2 framework. This module allows you to load and use any HuggingFace model with fairseq2's training and inference pipelines.
+
+API Reference
+-------------
+
+High-Level API
+~~~~~~~~~~~~~~
+
+.. autofunction:: fairseq2.models.hg.load_hg_model_simple
+
+.. autofunction:: fairseq2.models.hg.load_hg_tokenizer_simple
+
+Convenience Functions
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autofunction:: fairseq2.models.hg.load_causal_lm
+
+.. autofunction:: fairseq2.models.hg.load_seq2seq_lm
+
+.. autofunction:: fairseq2.models.hg.load_multimodal_model
+
+Configuration
+~~~~~~~~~~~~~
+
+.. autoclass:: fairseq2.models.hg.HuggingFaceModelConfig
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+.. autoclass:: fairseq2.models.hg.HgTokenizerConfig
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Factory Functions
+~~~~~~~~~~~~~~~~~
+
+.. autofunction:: fairseq2.models.hg.create_hg_model
+
+.. autofunction:: fairseq2.models.hg.register_hg_model_class
+
+Tokenizer Classes
+~~~~~~~~~~~~~~~~~
+
+.. autoclass:: fairseq2.models.hg.HgTokenizer
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+.. autofunction:: fairseq2.models.hg.load_hg_tokenizer
+
+Hub Integration
+~~~~~~~~~~~~~~~
+
+.. autofunction:: fairseq2.models.hg.get_hg_model_hub
+
+.. autofunction:: fairseq2.models.hg.get_hg_tokenizer_hub
+
+Exceptions
+~~~~~~~~~~
+
+.. autoexception:: fairseq2.models.hg.HuggingFaceModelError
+    :members:
+    :undoc-members:
+
+
+Examples
+--------
+
+Basic Model Loading
+~~~~~~~~~~~~~~~~~~~
+
+Use DialoGPT for conversational AI:
+
+.. code-block:: python
+
+    from fairseq2.models.hg import load_causal_lm, load_hg_tokenizer_simple
+    import torch
+
+    # Load DialoGPT model
+    model = load_causal_lm("microsoft/DialoGPT-small")
+    tokenizer = load_hg_tokenizer_simple("microsoft/DialoGPT-small")
+
+    # Conversation
+    user_input = "How are you doing today?"
+    inputs = tokenizer.encode(user_input + tokenizer.eos_token).unsqueeze(0)
+
+    with torch.no_grad():
+        outputs = model.generate(
+            inputs,
+            max_length=inputs.shape[1] + 20,
+            num_return_sequences=1,
+            pad_token_id=tokenizer.vocab_info.eos_idx,
+            do_sample=True,
+            temperature=0.7,
+        )
+
+response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+print(f"Bot: {response[len(user_input):]}")
+
+Sequence-to-Sequence Tasks
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Use T5 for translation and summarization:
+
+.. code-block:: python
+
+    from fairseq2.models.hg import load_seq2seq_lm, load_hg_tokenizer_simple
+    import torch
+
+    # Load T5 model
+    model = load_seq2seq_lm("t5-small")
+    tokenizer = load_hg_tokenizer_simple("t5-small")
+
+    # Translation task
+    text = "translate English to French: Hello, how are you?"
+    inputs = tokenizer.encode(text).unsqueeze(0)
+
+    with torch.no_grad():
+        outputs = model.generate(inputs, max_length=50)
+
+    translation = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    print(f"Translation: {translation}")
+
+
+Custom Model Registration
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Register custom models not supported by Auto classes:
+
+.. code-block:: python
+
+    from fairseq2.models.hg import register_hg_model_class, load_hg_model_simple
+
+    # Register a custom model class
+    register_hg_model_class(
+        config_class_name="Qwen2_5OmniConfig",
+        model_class="Qwen2_5OmniForConditionalGeneration",
+        processor_class="Qwen2_5OmniProcessor",
+    )
+
+    # Now load the custom model
+    model = load_hg_model_simple(
+        "Qwen/Qwen2.5-Omni-3B",
+        model_type="custom",
+        use_processor=True,
+        trust_remote_code=True,
+    )
+
+Hub Loading
+~~~~~~~~~~~
+
+Load models/tokenizers using the fairseq2 hub system:
+
+.. code-block:: python
+
+    from fairseq2.models.hg import get_hg_model_hub, get_hg_tokenizer_hub
+
+    name = "hg_qwen25_omni_3b"
+
+    # Load a pre-configured model
+    model_hub = get_hg_model_hub()
+    model = model_hub.load_model(name)
+
+    # Load the corresponding tokenizer
+    tokenizer_hub = get_hg_tokenizer_hub()
+    tokenizer = tokenizer_hub.load_tokenizer(name)
+
+
+.. note::
+    This module requires the ``transformers`` library. Install it with:
+    ``pip install transformers``
+
+.. warning::
+    Some models require ``trust_remote_code=True`` for custom architectures.
+    Only use this with trusted model sources.
+
+
+Module Structure
+----------------
+
+.. autosummary::
+   :toctree: generated/
+   :nosignatures:
+
+   fairseq2.models.hg.api
+   fairseq2.models.hg.config
+   fairseq2.models.hg.factory
+   fairseq2.models.hg.hub
+   fairseq2.models.hg.tokenizer

doc/source/reference/api/models/index.rst

@@ -14,6 +14,7 @@ The models module includes:
 - Mistral models
 - Wav2Vec2 models
 - NLLB translation models
+- HuggingFace Transformers integration
 - Model loading and configuration utilities
 - Model hub interface for advanced operations
 
@@ -23,6 +24,7 @@ The models module includes:
    hub
    qwen
    llama
+   ../fairseq2.models.hg
 
 Quick Start
 -----------

setup.py

@@ -79,6 +79,11 @@
             "polars>=1.19.0",
             "xxhash~=3.5",
         ],
+        "hg": [
+            "accelerate>=1.10.0",
+            "transformers>=4.56.1",
+            "qwen-omni-utils~=0.0.8",
+        ],
     },
     entry_points={"console_scripts": ["fairseq2=fairseq2.cli:main"]},
 )

src/fairseq2/assets/cards/models/hg.yaml

@@ -0,0 +1,59 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+
+name: hg_qwen25_omni_3b
+model_family: hg
+model_arch: custom
+model_config:
+  hf_name: Qwen/Qwen2.5-Omni-3B
+  use_processor: true
+  custom_model_class: Qwen2_5OmniForConditionalGeneration
+  custom_processor_class: Qwen2_5OmniProcessor
+  trust_remote_code: true
+checkpoint: hg://Qwen/Qwen2.5-Omni-3B
+tokenizer: hg://Qwen/Qwen2.5-Omni-3B
+tokenizer_family: hg
+
+---
+
+name: hg_qwen25_omni_7b
+model_family: hg
+model_arch: custom
+model_config:
+  hf_name: Qwen/Qwen2.5-Omni-7B
+  use_processor: true
+  custom_model_class: Qwen2_5OmniForConditionalGeneration
+  custom_processor_class: Qwen2_5OmniProcessor
+  trust_remote_code: true
+checkpoint: hg://Qwen/Qwen2.5-Omni-7B
+tokenizer: hg://Qwen/Qwen2.5-Omni-7B
+tokenizer_family: hg
+
+---
+
+name: hg_qwen3_0.6b
+model_family: hg
+model_arch: causal_lm
+model_config:
+  hf_name: Qwen/Qwen3-0.6B
+  trust_remote_code: true
+checkpoint: hg://Qwen/Qwen3-0.6B
+tokenizer: hg://Qwen/Qwen3-0.6B
+tokenizer_family: hg
+
+---
+
+name: hg_qwen3_1.7b
+model_family: hg
+model_arch: causal_lm
+model_config:
+  hf_name: Qwen/Qwen3-1.7B
+  trust_remote_code: true
+checkpoint: hg://Qwen/Qwen3-1.7B
+tokenizer: hg://Qwen/Qwen3-1.7B
+tokenizer_family: hg

src/fairseq2/assets/download_manager.py

@@ -263,7 +263,7 @@ def download_model(
             path = snapshot_download(
                 repo_id=repo_id,
                 repo_type="model",
-                allow_patterns="*.safetensors",
+                allow_patterns=["*.safetensors*", "*.pt"],
                 force_download=force,
             )
         except HfHubHTTPError as ex:

src/fairseq2/composition/models.py

@@ -22,6 +22,12 @@
     ShardSpecsProvider,
     StandardModelFamily,
 )
+from fairseq2.models.hg import (
+    HG_FAMILY,
+    HuggingFaceModelConfig,
+    create_hg_model,
+    register_hg_configs,
+)
 from fairseq2.models.jepa import (
     JEPA_FAMILY,
     JepaConfig,
@@ -358,3 +364,23 @@ def _register_model_families(container: DependencyContainer) -> None:
     )
 
     register_wav2vec2_asr_configs(container)
+
+    # HuggingFace
+    hg_kls: type[Module] | type[PreTrainedModel]
+    try:
+        from transformers import PreTrainedModel
+
+        hg_kls = PreTrainedModel
+    except ImportError:
+        hg_kls = Module
+
+    register_model_family(
+        container,
+        HG_FAMILY,
+        kls=hg_kls,
+        config_kls=HuggingFaceModelConfig,
+        factory=create_hg_model,
+        supports_meta=False,  # HF models don't support meta device initialization
+    )
+
+    register_hg_configs(container)