From dc03a60124c38dafd1d2885ffbb1dac0525bf919 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Mon, 28 Aug 2023 16:16:48 -0500 Subject: [PATCH 01/21] Install espeak --- .github/workflows/windows.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 439e3dc..dbcd59c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -30,6 +30,8 @@ jobs: run: cd espeak-ng && cmake -Bbuild -DUSE_ASYNC:BOOL=OFF -DBUILD_SHARED_LIBS:BOOL=${{ matrix.shlib }} -A ${{ matrix.arch }} -DUSE_MBROLA:BOOL=OFF -DUSE_LIBSONIC:BOOL=OFF -DUSE_LIBPCAUDIO:BOOL=OFF -DUSE_KLATT:BOOL=OFF -DUSE_SPEECHPLAYER:BOOL=OFF -DEXTRA_cmn:BOOL=ON -DEXTRA_ru:BOOL=ON - name: make-espeak run: cd espeak-ng && cmake --build build --config ${{ matrix.config }} + - name: install-espeak + run: cd espeak-ng && cmake --install build --prefix "$PWD" - uses: actions/checkout@v3 with: path: piper-phonemize From 6806d21fb974684651c24cc103bc1da3699e1b78 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Mon, 28 Aug 2023 16:19:59 -0500 Subject: [PATCH 02/21] Use full path --- .github/workflows/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index dbcd59c..0e0318a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -39,7 +39,7 @@ jobs: run: curl -L -o onnxruntime.zip 'https://github.com/microsoft/onnxruntime/releases/download/v1.14.1/onnxruntime-win-x64-1.14.1.zip' && unzip onnxruntime.zip && mkdir -p piper-phonemize/lib/Windows-x86_64 && mv onnxruntime-* piper-phonemize/lib/Windows-x86_64/onnxruntime - name: configure-piper env: - PKG_CONFIG_PATH: espeak-ng/lib/pkgconfig + PKG_CONFIG_PATH: "$PWD/espeak-ng/lib/pkgconfig" run: cd piper-phonemize && cmake -Bbuild - name: make-piper run: cd piper-phonemize && cmake --build build --config ${{ matrix.config }} From b32bca7c921471efc9d0dc6eac9e1602a6833197 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Mon, 28 Aug 2023 16:23:55 -0500 Subject: [PATCH 03/21] Try again --- .github/workflows/windows.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 0e0318a..71ae1b1 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -38,8 +38,6 @@ jobs: - name: prepare-onnxruntime run: curl -L -o onnxruntime.zip 'https://github.com/microsoft/onnxruntime/releases/download/v1.14.1/onnxruntime-win-x64-1.14.1.zip' && unzip onnxruntime.zip && mkdir -p piper-phonemize/lib/Windows-x86_64 && mv onnxruntime-* piper-phonemize/lib/Windows-x86_64/onnxruntime - name: configure-piper - env: - PKG_CONFIG_PATH: "$PWD/espeak-ng/lib/pkgconfig" - run: cd piper-phonemize && cmake -Bbuild + run: cd piper-phonemize && PKG_CONFIG_PATH="${PWD}/espeak-ng/lib/pkgconfig" cmake -Bbuild - name: make-piper run: cd piper-phonemize && cmake --build build --config ${{ matrix.config }} From de5da01c6deb1c92161ad1732dcf2085bbb570b8 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Mon, 28 Aug 2023 16:28:32 -0500 Subject: [PATCH 04/21] And again --- .github/workflows/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 71ae1b1..bbd7f48 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -38,6 +38,6 @@ jobs: - name: prepare-onnxruntime run: curl -L -o onnxruntime.zip 'https://github.com/microsoft/onnxruntime/releases/download/v1.14.1/onnxruntime-win-x64-1.14.1.zip' && unzip onnxruntime.zip && mkdir -p piper-phonemize/lib/Windows-x86_64 && mv onnxruntime-* piper-phonemize/lib/Windows-x86_64/onnxruntime - name: configure-piper - run: cd piper-phonemize && PKG_CONFIG_PATH="${PWD}/espeak-ng/lib/pkgconfig" cmake -Bbuild + run: cd piper-phonemize && cmake -Bbuild -DPKG_CONFIG_PATH="${PWD}/espeak-ng/lib/pkgconfig" - name: make-piper run: cd piper-phonemize && cmake --build build --config ${{ matrix.config }} From 77ff692d86213a210906ca0fc92f51559864d46d Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Mon, 28 Aug 2023 16:34:35 -0500 Subject: [PATCH 05/21] Again --- .github/workflows/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index bbd7f48..0514de0 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -38,6 +38,6 @@ jobs: - name: prepare-onnxruntime run: curl -L -o onnxruntime.zip 'https://github.com/microsoft/onnxruntime/releases/download/v1.14.1/onnxruntime-win-x64-1.14.1.zip' && unzip onnxruntime.zip && mkdir -p piper-phonemize/lib/Windows-x86_64 && mv onnxruntime-* piper-phonemize/lib/Windows-x86_64/onnxruntime - name: configure-piper - run: cd piper-phonemize && cmake -Bbuild -DPKG_CONFIG_PATH="${PWD}/espeak-ng/lib/pkgconfig" + run: cd piper-phonemize && cmake -Bbuild -DCMAKE_PREFIX_PATH="${PWD}/espeak-ng/lib/pkgconfig" - name: make-piper run: cd piper-phonemize && cmake --build build --config ${{ matrix.config }} From 78e3c04f239cdf493e2e9923aa9ea9372f3895ed Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Mon, 28 Aug 2023 16:40:17 -0500 Subject: [PATCH 06/21] Again --- .github/workflows/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 0514de0..1dd30e6 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -38,6 +38,6 @@ jobs: - name: prepare-onnxruntime run: curl -L -o onnxruntime.zip 'https://github.com/microsoft/onnxruntime/releases/download/v1.14.1/onnxruntime-win-x64-1.14.1.zip' && unzip onnxruntime.zip && mkdir -p piper-phonemize/lib/Windows-x86_64 && mv onnxruntime-* piper-phonemize/lib/Windows-x86_64/onnxruntime - name: configure-piper - run: cd piper-phonemize && cmake -Bbuild -DCMAKE_PREFIX_PATH="${PWD}/espeak-ng/lib/pkgconfig" + run: cd piper-phonemize && env:PKG_CONFIG_PATH="${PWD}/espeak-ng/lib/pkgconfig" && cmake -Bbuild - name: make-piper run: cd piper-phonemize && cmake --build build --config ${{ matrix.config }} From d30600b2178f8397a5c1aac7fab6ee005cf9ba3b Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Thu, 31 Aug 2023 12:05:06 -0500 Subject: [PATCH 07/21] Working on Windows support --- CMakeLists.txt | 34 ++++++++++++++++++++++++++++++++-- Makefile | 17 ++++++++++++++--- src/main.cpp | 9 +++++++++ src/phoneme_ids.cpp | 8 ++++---- src/phoneme_ids.hpp | 9 +++++---- src/phonemize.cpp | 10 ++++++---- src/phonemize.hpp | 12 ++++++++---- src/tashkeel.cpp | 14 +++++++++++--- src/tashkeel.hpp | 6 ++++-- 9 files changed, 93 insertions(+), 26 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ad6fce5..4488c6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,18 +10,47 @@ project( LANGUAGES CXX ) -string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra -Wl,-rpath,'$ORIGIN'") +if(WIN32) + +# Force compiler to use UTF-8 for IPA constants +add_compile_options("$<$:/utf-8>") +add_compile_options("$<$:/utf-8>") + +else() + +# Linux flags +string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra -Wl,-rpath,'$$ORIGIN' -I/usr/include/espeak-ng") string(APPEND CMAKE_C_FLAGS " -Wall -Wextra") +endif() + # lib/Linux-x86_64 # lib/Linux-aarch64 -set(ONNXRUNTIME_ROOTDIR ${CMAKE_CURRENT_LIST_DIR}/lib/${CMAKE_HOST_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}/onnxruntime) +# lib/Windows-AMD64 +if(NOT DEFINED ONNXRUNTIME_ROOTDIR) + +# Look for onnxruntime files in /lib +set(ONNXRUNTIME_ROOTDIR ${CMAKE_CURRENT_LIST_DIR}/lib/${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}/onnxruntime) + +endif() # ---- espeak-ng --- +if(DEFINED ESPEAK_NG_DIR) + +# Expect -DESPEAK_NG_DIR on Windows +set(ESPEAK_NG_INCLUDE_DIRS "${ESPEAK_NG_DIR}/include") +set(ESPEAK_NG_LIBRARY_DIRS "${ESPEAK_NG_DIR}/lib") +set(ESPEAK_NG_LIBRARIES "espeak-ng") + +else() + +# Locate espeak-ng with pkg-config find_package(PkgConfig) pkg_check_modules(ESPEAK_NG REQUIRED espeak-ng<2) +endif() + # ---- Declare library ---- add_library( @@ -29,6 +58,7 @@ add_library( src/phonemize.cpp src/phoneme_ids.cpp src/tashkeel.cpp + src/shared.cpp ) set_target_properties(piper_phonemize PROPERTIES diff --git a/Makefile b/Makefile index cb4f784..6ef9d5b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: release test python python-test docker +.PHONY: release release-windows test python python-test docker docker-windows LIB_DIR := lib/Linux-$(shell uname -m) DOCKER_PLATFORM ?= linux/amd64,linux/arm64,linux/arm/v7 @@ -6,14 +6,22 @@ VENV ?= .venv release: mkdir -p build - cd build && PKG_CONFIG_PATH='../espeak-ng/build/lib/pkgconfig' cmake .. -DCMAKE_BUILD_TYPE=Release && make + cd build && cmake .. -DCMAKE_BUILD_TYPE=Release && make cp -a espeak-ng/build/lib/libespeak*.so* build/ cp -R espeak-ng/build/share/espeak-ng-data build/ cp -a $(LIB_DIR)/onnxruntime/lib/libonnxruntime*.so* build/ cp etc/libtashkeel_model.ort build/ +release-windows: + mkdir -p build + cd build && cmake .. -DCMAKE_BUILD_TYPE=Release && make + cp -a espeak-ng/build_windows/lib/libespeak*.dll build/ + cp -R espeak-ng/build/share/espeak-ng-data build/ + cp -a $(LIB_DIR)/onnxruntime/lib/libonnxruntime*.dll build/ + cp etc/libtashkeel_model.ort build/ + test: - g++ -Wall -o test -Iespeak-ng/build/include -Lespeak-ng/build/lib -I$(LIB_DIR)/onnxruntime/include -L$(LIB_DIR)/onnxruntime/lib -Isrc -std=c++17 src/test.cpp src/phonemize.cpp src/phoneme_ids.cpp src/tashkeel.cpp -lespeak-ng -lonnxruntime + g++ -Wall -o test -Iespeak-ng/build/include -Lespeak-ng/build/lib -I$(LIB_DIR)/onnxruntime/include -L$(LIB_DIR)/onnxruntime/lib -Isrc -std=c++17 src/test.cpp src/phonemize.cpp src/phoneme_ids.cpp src/tashkeel.cpp src/shared.cpp -lespeak-ng -lonnxruntime LD_LIBRARY_PATH="espeak-ng/build/lib:$(LIB_DIR)/onnxruntime/lib:${LD_LIBRARY_PATH}" ./test 'espeak-ng/build/share/espeak-ng-data' python: @@ -31,3 +39,6 @@ python-wheel: docker: docker buildx build . --platform "$(DOCKER_PLATFORM)" --output 'type=local,dest=dist' + +docker-windows: + docker buildx build . -f Dockerfile.windows --output 'type=local,dest=dist/windows_amd64' diff --git a/src/main.cpp b/src/main.cpp index 46799f7..9a31912 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -7,6 +7,10 @@ #include +#ifdef _WIN32 +#include +#endif + #include "json.hpp" #include "phoneme_ids.hpp" #include "phonemize.hpp" @@ -40,6 +44,11 @@ int main(int argc, char *argv[]) { RunConfig runConfig; parseArgs(argc, argv, runConfig); +#ifdef _WIN32 + // Required on Windows to show IPA symbols + SetConsoleOutputCP(CP_UTF8); +#endif + piper::eSpeakPhonemeConfig eSpeakConfig; piper::CodepointsPhonemeConfig codepointsConfig; piper::PhonemeIdConfig idConfig; diff --git a/src/phoneme_ids.cpp b/src/phoneme_ids.cpp index 6372dc0..760b04a 100644 --- a/src/phoneme_ids.cpp +++ b/src/phoneme_ids.cpp @@ -198,10 +198,10 @@ std::map DEFAULT_ALPHABET = { {U'—', {48}}, }}}; -void phonemes_to_ids(const std::vector &phonemes, - PhonemeIdConfig &config, - std::vector &phonemeIds, - std::map &missingPhonemes) { +PIPERPHONEMIZE_EXPORT void +phonemes_to_ids(const std::vector &phonemes, PhonemeIdConfig &config, + std::vector &phonemeIds, + std::map &missingPhonemes) { auto phonemeIdMap = std::make_shared(DEFAULT_PHONEME_ID_MAP); if (config.phonemeIdMap) { diff --git a/src/phoneme_ids.hpp b/src/phoneme_ids.hpp index 2ac2dc8..0fe3cea 100644 --- a/src/phoneme_ids.hpp +++ b/src/phoneme_ids.hpp @@ -6,6 +6,7 @@ #include #include "phonemize.hpp" +#include "shared.hpp" namespace piper { @@ -35,10 +36,10 @@ extern const size_t MAX_PHONEMES; extern PhonemeIdMap DEFAULT_PHONEME_ID_MAP; extern std::map DEFAULT_ALPHABET; -void phonemes_to_ids(const std::vector &phonemes, - PhonemeIdConfig &config, - std::vector &phonemeIds, - std::map &missingPhonemes); +PIPERPHONEMIZE_EXPORT void +phonemes_to_ids(const std::vector &phonemes, PhonemeIdConfig &config, + std::vector &phonemeIds, + std::map &missingPhonemes); } // namespace piper diff --git a/src/phonemize.cpp b/src/phonemize.cpp index 28b01e4..c920a95 100644 --- a/src/phonemize.cpp +++ b/src/phonemize.cpp @@ -14,8 +14,9 @@ namespace piper { std::map DEFAULT_PHONEME_MAP = { {"pt-br", {{U'c', {U'k'}}}}}; -void phonemize_eSpeak(std::string text, eSpeakPhonemeConfig &config, - std::vector> &phonemes) { +PIPERPHONEMIZE_EXPORT void +phonemize_eSpeak(std::string text, eSpeakPhonemeConfig &config, + std::vector> &phonemes) { auto voice = config.voice; int result = espeak_SetVoiceByName(voice.c_str()); @@ -134,8 +135,9 @@ void phonemize_eSpeak(std::string text, eSpeakPhonemeConfig &config, // ---------------------------------------------------------------------------- -void phonemize_codepoints(std::string text, CodepointsPhonemeConfig &config, - std::vector> &phonemes) { +PIPERPHONEMIZE_EXPORT void +phonemize_codepoints(std::string text, CodepointsPhonemeConfig &config, + std::vector> &phonemes) { if (config.casing == CASING_LOWER) { text = una::cases::to_lowercase_utf8(text); diff --git a/src/phonemize.hpp b/src/phonemize.hpp index bb8f4d9..5e4238e 100644 --- a/src/phonemize.hpp +++ b/src/phonemize.hpp @@ -6,6 +6,8 @@ #include #include +#include "shared.hpp" + #define CLAUSE_INTONATION_FULL_STOP 0x00000000 #define CLAUSE_INTONATION_COMMA 0x00001000 #define CLAUSE_INTONATION_QUESTION 0x00002000 @@ -48,8 +50,9 @@ struct eSpeakPhonemeConfig { // Returns phonemes for each sentence as a separate std::vector. // // Assumes espeak_Initialize has already been called. -void phonemize_eSpeak(std::string text, eSpeakPhonemeConfig &config, - std::vector> &phonemes); +PIPERPHONEMIZE_EXPORT void +phonemize_eSpeak(std::string text, eSpeakPhonemeConfig &config, + std::vector> &phonemes); enum TextCasing { CASING_IGNORE = 0, @@ -68,8 +71,9 @@ struct CodepointsPhonemeConfig { // Returns a single std::vector of "phonemes". // // Does not detect sentence boundaries. -void phonemize_codepoints(std::string text, CodepointsPhonemeConfig &config, - std::vector> &phonemes); +PIPERPHONEMIZE_EXPORT void +phonemize_codepoints(std::string text, CodepointsPhonemeConfig &config, + std::vector> &phonemes); } // namespace piper diff --git a/src/tashkeel.cpp b/src/tashkeel.cpp index 2db274e..23683b7 100644 --- a/src/tashkeel.cpp +++ b/src/tashkeel.cpp @@ -78,15 +78,23 @@ std::set HARAKAT_CHARS{ std::set INVALID_HARAKA_IDS{UNK_ID, 8}; -void tashkeel_load(std::string modelPath, State &state) { +PIPERPHONEMIZE_EXPORT void tashkeel_load(std::string modelPath, State &state) { state.env = Ort::Env(OrtLoggingLevel::ORT_LOGGING_LEVEL_WARNING, instanceName.c_str()); state.env.DisableTelemetryEvents(); state.options.SetExecutionMode(ExecutionMode::ORT_PARALLEL); - state.onnx = Ort::Session(state.env, modelPath.c_str(), state.options); + +#ifdef _WIN32 + auto modelPathW = std::wstring(modelPath.begin(), modelPath.end()); + auto modelPathStr = modelPathW.c_str(); +#else + auto modelPathStr = modelPath.c_str(); +#endif + + state.onnx = Ort::Session(state.env, modelPathStr, state.options); } -std::string tashkeel_run(std::string text, State &state) { +PIPERPHONEMIZE_EXPORT std::string tashkeel_run(std::string text, State &state) { auto memoryInfo = Ort::MemoryInfo::CreateCpu( OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault); diff --git a/src/tashkeel.hpp b/src/tashkeel.hpp index 4f00e57..1611779 100644 --- a/src/tashkeel.hpp +++ b/src/tashkeel.hpp @@ -7,6 +7,8 @@ #include +#include "shared.hpp" + // https://github.com/mush42/libtashkeel namespace tashkeel { @@ -29,8 +31,8 @@ struct State { State() : onnx(nullptr){}; }; -void tashkeel_load(std::string modelPath, State &state); -std::string tashkeel_run(std::string text, State &state); +PIPERPHONEMIZE_EXPORT void tashkeel_load(std::string modelPath, State &state); +PIPERPHONEMIZE_EXPORT std::string tashkeel_run(std::string text, State &state); } // namespace tashkeel From 82a56b8fef7b3c6ef50763e2e7bcdf89e7dfd457 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Thu, 7 Sep 2023 14:08:18 -0500 Subject: [PATCH 08/21] Add espeak-ng submodule --- espeak-ng | 1 + 1 file changed, 1 insertion(+) create mode 160000 espeak-ng diff --git a/espeak-ng b/espeak-ng new file mode 160000 index 0000000..6fd1834 --- /dev/null +++ b/espeak-ng @@ -0,0 +1 @@ +Subproject commit 6fd1834efbb19c28f0bad5328b409ce00582fee1 From 93a02a6aa8d6e39af44e126bc6503a638d69faaa Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Thu, 7 Sep 2023 14:08:35 -0500 Subject: [PATCH 09/21] Adding Windows support --- CMakeLists.txt | 2 +- src/main.cpp | 4 +++- src/shared.cpp | 1 + src/shared.hpp | 11 +++++++++++ 4 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 src/shared.cpp create mode 100644 src/shared.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4488c6b..582c7e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,7 @@ add_compile_options("$<$:/utf-8>") else() # Linux flags -string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra -Wl,-rpath,'$$ORIGIN' -I/usr/include/espeak-ng") +string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra -Wl,-rpath,'$$ORIGIN'") string(APPEND CMAKE_C_FLAGS " -Wall -Wextra") endif() diff --git a/src/main.cpp b/src/main.cpp index 9a31912..b8ef050 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -7,7 +7,9 @@ #include -#ifdef _WIN32 +#ifdef _MSC_VER +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX #include #endif diff --git a/src/shared.cpp b/src/shared.cpp new file mode 100644 index 0000000..41d7c6c --- /dev/null +++ b/src/shared.cpp @@ -0,0 +1 @@ +#include "shared.hpp" diff --git a/src/shared.hpp b/src/shared.hpp new file mode 100644 index 0000000..c7264f0 --- /dev/null +++ b/src/shared.hpp @@ -0,0 +1,11 @@ +#ifndef SHARED_H_ +#define SHARED_H_ + +#ifdef _WIN32 +#define PIPERPHONEMIZE_EXPORT __declspec(dllexport) +#else +#define PIPERPHONEMIZE_EXPORT +#endif + + +#endif // SHARED_H_ From 30349d0c52c6e23434d72c6c1dd02646cbcb18d5 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Thu, 7 Sep 2023 17:35:09 -0500 Subject: [PATCH 10/21] Try workflow --- .github/workflows/main.yml | 90 ++++++++++++++++++++++++ .github/workflows/windows.yml | 43 ------------ .gitignore | 4 +- CMakeLists.txt | 129 ++++++++++++++++++++++++++-------- 4 files changed, 192 insertions(+), 74 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 .github/workflows/windows.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..9949139 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,90 @@ +name: main + +on: + workflow_dispatch: + push: + tags: + - "*" + pull_request: + branches: [ master ] + +jobs: + create_release: + name: Create release + runs-on: ubuntu-latest + outputs: + upload_url: ${{ steps.create_release.outputs.upload_url }} + steps: + - name: Create release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + tag_name: ${{ github.ref }} + release_name: ${{ github.ref }} + draft: false + prerelease: false + build_linux: + name: "linux build: ${{ matrix.arch }}" + runs-on: ubuntu-20.04 # use older version on purpose for GLIBC + needs: create_release # we need to know the upload URL + strategy: + fail-fast: true + matrix: + arch: [x64, aarch64, armv7] + steps: + - uses: actions/checkout@v3 + - name: configure + run: | + cmake -Bbuild -DCMAKE_INSTALL_PREFIX=_install/piper-phonemize + - name: build + run: | + cmake --build build --config Release + - name: install + run: | + cmake --install build + - name: package + run: | + cd _install && \ + tar -czf piper-phonemize_linux_${{ matrix.arch }}.tar.gz piper-phonemize/ + - name: upload + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: _install/piper-phonemize_linux_${{ matrix.arch }}.tar.gz + asset_name: piper-phonemize_linux_${{ matrix.arch }}.tar.gz + asset_content_type: application/octet-stream + build_windows: + runs-on: windows-latest + name: "windows build: ${{ matrix.arch }}" + strategy: + fail-fast: false + matrix: + arch: [x64] + steps: + - uses: actions/checkout@v3 + - name: configure + run: | + cmake -Bbuild -DCMAKE_INSTALL_PREFIX=_install/piper-phonemize + - name: build + run: | + cmake --build build --config Release + - name: install + run: | + cmake --install build + - name: package + run: | + cd _install + Compress-Archive -LiteralPath espeak-ng -DestinationPath piper-phonemize_windows_amd64.zip + - name: upload + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: _install/piper-phonemize_windows_amd64.zip + asset_name: piper-phonemize_windows_amd64.zip + asset_content_type: application/zip diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml deleted file mode 100644 index 1dd30e6..0000000 --- a/.github/workflows/windows.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Windows - -on: - workflow_dispatch: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - runs-on: windows-latest - name: "build: ${{ matrix.arch }} ${{ matrix.config }} ${{ matrix.link }}" - strategy: - fail-fast: false - matrix: - arch: [x64] - config: [Release] - link: [dll] - - include: - - link: dll - shlib: "ON" - steps: - - uses: actions/checkout@v3 - with: - repository: rhasspy/espeak-ng - path: espeak-ng - - name: configure-espeak - run: cd espeak-ng && cmake -Bbuild -DUSE_ASYNC:BOOL=OFF -DBUILD_SHARED_LIBS:BOOL=${{ matrix.shlib }} -A ${{ matrix.arch }} -DUSE_MBROLA:BOOL=OFF -DUSE_LIBSONIC:BOOL=OFF -DUSE_LIBPCAUDIO:BOOL=OFF -DUSE_KLATT:BOOL=OFF -DUSE_SPEECHPLAYER:BOOL=OFF -DEXTRA_cmn:BOOL=ON -DEXTRA_ru:BOOL=ON - - name: make-espeak - run: cd espeak-ng && cmake --build build --config ${{ matrix.config }} - - name: install-espeak - run: cd espeak-ng && cmake --install build --prefix "$PWD" - - uses: actions/checkout@v3 - with: - path: piper-phonemize - - name: prepare-onnxruntime - run: curl -L -o onnxruntime.zip 'https://github.com/microsoft/onnxruntime/releases/download/v1.14.1/onnxruntime-win-x64-1.14.1.zip' && unzip onnxruntime.zip && mkdir -p piper-phonemize/lib/Windows-x86_64 && mv onnxruntime-* piper-phonemize/lib/Windows-x86_64/onnxruntime - - name: configure-piper - run: cd piper-phonemize && env:PKG_CONFIG_PATH="${PWD}/espeak-ng/lib/pkgconfig" && cmake -Bbuild - - name: make-piper - run: cd piper-phonemize && cmake --build build --config ${{ matrix.config }} diff --git a/.gitignore b/.gitignore index 4cb8fb4..3422681 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,5 @@ htmlcov /*.so /test -espeak-ng-data/ - -/espeak-ng/ /lib/ +/download/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 582c7e7..98c14ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,44 +11,114 @@ project( ) if(WIN32) - -# Force compiler to use UTF-8 for IPA constants -add_compile_options("$<$:/utf-8>") -add_compile_options("$<$:/utf-8>") + # Force compiler to use UTF-8 for IPA constants + add_compile_options("$<$:/utf-8>") + add_compile_options("$<$:/utf-8>") else() - -# Linux flags -string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra -Wl,-rpath,'$$ORIGIN'") -string(APPEND CMAKE_C_FLAGS " -Wall -Wextra") - + # Linux flags + string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra -Wl,-rpath,'$$ORIGIN'") + string(APPEND CMAKE_C_FLAGS " -Wall -Wextra") endif() -# lib/Linux-x86_64 -# lib/Linux-aarch64 -# lib/Windows-AMD64 -if(NOT DEFINED ONNXRUNTIME_ROOTDIR) +# ---- onnxruntime --- # Look for onnxruntime files in /lib -set(ONNXRUNTIME_ROOTDIR ${CMAKE_CURRENT_LIST_DIR}/lib/${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}/onnxruntime) - +if(NOT DEFINED ONNXRUNTIME_ROOTDIR) + if(NOT DEFINED ONNXRUNTIME_VERSION) + set(ONNXRUNTIME_VERSION "1.14.1") + endif() + + if(WIN32) + # Windows x86-64 + set(ONNXRUNTIME_PREFIX "onnxruntime-win-x64-${ONNXRUNTIME_VERSION}") + set(ONNXRUNTIME_EXT "zip") + else() + if(CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64) + # Linux x86-64 + set(ONNXRUNTIME_PREFIX "onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}") + elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) + # Linux ARM 64-bit + set(ONNXRUNTIME_PREFIX "onnxruntime-linux-aarch64-${ONNXRUNTIME_VERSION}") + elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l) + # Linux ARM 32-bit + set(ONNXRUNTIME_PREFIX "onnxruntime-linux-arm32-${ONNXRUNTIME_VERSION}") + set(ONNXRUNTIME_URL "https://github.com/synesthesiam/prebuilt-apps/releases/download/v1.0/onnxruntime-linux-arm32-${ONNXRUNTIME_VERSION}.tgz") + else() + message(FATAL_ERROR "Unsupported architecture for onnxruntime") + endif() + + set(ONNXRUNTIME_EXT "tgz") + endif() + + if(NOT DEFINED ONNXRUNTIME_URL) + set(ONNXRUNTIME_URL "https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/${ONNXRUNTIME_PREFIX}.${ONNXRUNTIME_EXT}") + endif() + + set(ONNXRUNTIME_FILENAME "${ONNXRUNTIME_PREFIX}.${ONNXRUNTIME_EXT}") + set(ONNXRUNTIME_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/lib/${ONNXRUNTIME_PREFIX}") + + if(NOT EXISTS "${ONNXRUNTIME_ROOTDIR}") + if(NOT EXISTS "download/${ONNXRUNTIME_FILENAME}") + # Download onnxruntime release + message("Downloading ${ONNXRUNTIME_URL}") + file(DOWNLOAD "${ONNXRUNTIME_URL}" "download/${ONNXRUNTIME_FILENAME}") + endif() + + # Extract .zip or .tgz to a directory like lib/onnxruntime-linux-x64-1.14.1/ + file(ARCHIVE_EXTRACT INPUT "download/${ONNXRUNTIME_FILENAME}" DESTINATION "${CMAKE_CURRENT_LIST_DIR}/lib") + endif() endif() # ---- espeak-ng --- -if(DEFINED ESPEAK_NG_DIR) - -# Expect -DESPEAK_NG_DIR on Windows -set(ESPEAK_NG_INCLUDE_DIRS "${ESPEAK_NG_DIR}/include") -set(ESPEAK_NG_LIBRARY_DIRS "${ESPEAK_NG_DIR}/lib") -set(ESPEAK_NG_LIBRARIES "espeak-ng") - +if(DEFINED ESPEAK_NG_PKGCONFIG) + # Locate espeak-ng with pkg-config + find_package(PkgConfig) + pkg_check_modules(ESPEAK_NG REQUIRED espeak-ng<2) else() - -# Locate espeak-ng with pkg-config -find_package(PkgConfig) -pkg_check_modules(ESPEAK_NG REQUIRED espeak-ng<2) - + if(NOT DEFINED ESPEAK_NG_DIR) + # Download from https://github.com/rhasspy/espeak-ng + set(ESPEAK_NG_DIR "${CMAKE_CURRENT_LIST_DIR}/lib/espeak-ng") + if(NOT EXISTS "${ESPEAK_NG_DIR}") + if(NOT DEFINED ESPEAK_NG_RELEASE) + set(ESPEAK_NG_RELEASE "2023.9.7-4") + endif() + + if(WIN32) + # Windows x86-64 + set(ESPEAK_NG_FILENAME "windows_amd64.zip") + else() + if(CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64) + # Linux x86-64 + set(ESPEAK_NG_FILENAME "linux_x64.tar.gz") + elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) + # Linux ARM 64-bit + set(ESPEAK_NG_FILENAME "linux_aarch64.tar.gz") + elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l) + # Linux ARM 32-bit + set(ESPEAK_NG_FILENAME "linux_armv7.tar.gz") + else() + message(FATAL_ERROR "Unsupported architecture for rhasspy espeak-ng") + endif() + endif() + + set(ESPEAK_NG_ARCHIVE "espeak-ng_${ESPEAK_NG_RELEASE}_${ESPEAK_NG_FILENAME}") + + if(NOT EXISTS "download/${ESPEAK_NG_ARCHIVE}") + set(ESPEAK_NG_URL "https://github.com/rhasspy/espeak-ng/releases/download/${ESPEAK_NG_RELEASE}/${ESPEAK_NG_FILENAME}") + message("Downloading ${ESPEAK_NG_URL}") + file(DOWNLOAD "${ESPEAK_NG_URL}" "download/${ESPEAK_NG_ARCHIVE}") + endif() + + # Extract .zip or .tar.gz to lib/espeak-ng + file(ARCHIVE_EXTRACT INPUT "download/${ESPEAK_NG_ARCHIVE}" DESTINATION "${CMAKE_CURRENT_LIST_DIR}/lib") + endif() + endif() + + set(ESPEAK_NG_INCLUDE_DIRS "${ESPEAK_NG_DIR}/include") + set(ESPEAK_NG_LIBRARY_DIRS "${ESPEAK_NG_DIR}/lib") + set(ESPEAK_NG_LIBRARIES "espeak-ng") endif() # ---- Declare library ---- @@ -94,7 +164,10 @@ target_compile_features(piper_phonemize PUBLIC cxx_std_17) # ---- Declare executable ---- add_executable(piper_phonemize_exe src/main.cpp) -set_property(TARGET piper_phonemize_exe PROPERTY OUTPUT_NAME piper_phonemize) + +if(NOT WIN32) + set_property(TARGET piper_phonemize_exe PROPERTY OUTPUT_NAME piper_phonemize) +endif() target_compile_features(piper_phonemize_exe PUBLIC cxx_std_17) From 90280cc6e8cd266b7a67883dc5093370e9aa9143 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Thu, 7 Sep 2023 17:37:27 -0500 Subject: [PATCH 11/21] Fail fast --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9949139..9112fdd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -61,7 +61,7 @@ jobs: runs-on: windows-latest name: "windows build: ${{ matrix.arch }}" strategy: - fail-fast: false + fail-fast: true matrix: arch: [x64] steps: From 6c7a5e035efa32ccc5efafa974795c0b00acb614 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Thu, 7 Sep 2023 17:42:58 -0500 Subject: [PATCH 12/21] Need DEFAULT_ALPHABET --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 98c14ae..cb4604e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,7 +163,7 @@ target_compile_features(piper_phonemize PUBLIC cxx_std_17) # ---- Declare executable ---- -add_executable(piper_phonemize_exe src/main.cpp) +add_executable(piper_phonemize_exe src/main.cpp src/phoneme_ids.cpp) if(NOT WIN32) set_property(TARGET piper_phonemize_exe PROPERTY OUTPUT_NAME piper_phonemize) From 4304f49a77eb915ff0498ad28f8db9cc6533e5d5 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Thu, 7 Sep 2023 17:47:33 -0500 Subject: [PATCH 13/21] Fix dir name --- .github/workflows/main.yml | 2 +- .gitignore | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9112fdd..93d7403 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -78,7 +78,7 @@ jobs: - name: package run: | cd _install - Compress-Archive -LiteralPath espeak-ng -DestinationPath piper-phonemize_windows_amd64.zip + Compress-Archive -LiteralPath piper-phonemize -DestinationPath piper-phonemize_windows_amd64.zip - name: upload uses: actions/upload-release-asset@v1 env: diff --git a/.gitignore b/.gitignore index 3422681..b1ec3fc 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ htmlcov /lib/ /download/ +/cmake/ +/_install/ From f3210dc8df2b47bd168214f2869096c7d4df0d34 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Thu, 7 Sep 2023 17:50:12 -0500 Subject: [PATCH 14/21] Fix needs --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 93d7403..d1ac609 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,8 +5,6 @@ on: push: tags: - "*" - pull_request: - branches: [ master ] jobs: create_release: @@ -60,6 +58,7 @@ jobs: build_windows: runs-on: windows-latest name: "windows build: ${{ matrix.arch }}" + needs: create_release # we need to know the upload URL strategy: fail-fast: true matrix: From a5f481c77d5a4eda327ecb2b73417e2bf9da7a05 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Fri, 8 Sep 2023 16:30:55 -0500 Subject: [PATCH 15/21] Reformat --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cb4604e..7c2d6f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -183,8 +183,8 @@ target_link_directories( ) target_link_libraries(piper_phonemize_exe PUBLIC - piper_phonemize - ${ESPEAK_NG_LIBRARIES}) + piper_phonemize + ${ESPEAK_NG_LIBRARIES}) target_compile_options( piper_phonemize_exe PUBLIC From 17c6b89dd901e65985858e022d0080c7ed1cbbea Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Fri, 8 Sep 2023 17:03:56 -0500 Subject: [PATCH 16/21] Use external project instead for espeak-ng --- CMakeLists.txt | 112 +++++++++++++++++-------------------------------- 1 file changed, 38 insertions(+), 74 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c2d6f8..e37059f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,18 @@ project( LANGUAGES CXX ) +add_library( + piper_phonemize SHARED + src/phonemize.cpp + src/phoneme_ids.cpp + src/tashkeel.cpp + src/shared.cpp +) + +set_target_properties(piper_phonemize PROPERTIES + VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_VERSION_MAJOR}) + if(WIN32) # Force compiler to use UTF-8 for IPA constants add_compile_options("$<$:/utf-8>") @@ -72,93 +84,49 @@ endif() # ---- espeak-ng --- -if(DEFINED ESPEAK_NG_PKGCONFIG) - # Locate espeak-ng with pkg-config - find_package(PkgConfig) - pkg_check_modules(ESPEAK_NG REQUIRED espeak-ng<2) -else() - if(NOT DEFINED ESPEAK_NG_DIR) - # Download from https://github.com/rhasspy/espeak-ng - set(ESPEAK_NG_DIR "${CMAKE_CURRENT_LIST_DIR}/lib/espeak-ng") - if(NOT EXISTS "${ESPEAK_NG_DIR}") - if(NOT DEFINED ESPEAK_NG_RELEASE) - set(ESPEAK_NG_RELEASE "2023.9.7-4") - endif() - - if(WIN32) - # Windows x86-64 - set(ESPEAK_NG_FILENAME "windows_amd64.zip") - else() - if(CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64) - # Linux x86-64 - set(ESPEAK_NG_FILENAME "linux_x64.tar.gz") - elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) - # Linux ARM 64-bit - set(ESPEAK_NG_FILENAME "linux_aarch64.tar.gz") - elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l) - # Linux ARM 32-bit - set(ESPEAK_NG_FILENAME "linux_armv7.tar.gz") - else() - message(FATAL_ERROR "Unsupported architecture for rhasspy espeak-ng") - endif() - endif() - - set(ESPEAK_NG_ARCHIVE "espeak-ng_${ESPEAK_NG_RELEASE}_${ESPEAK_NG_FILENAME}") - - if(NOT EXISTS "download/${ESPEAK_NG_ARCHIVE}") - set(ESPEAK_NG_URL "https://github.com/rhasspy/espeak-ng/releases/download/${ESPEAK_NG_RELEASE}/${ESPEAK_NG_FILENAME}") - message("Downloading ${ESPEAK_NG_URL}") - file(DOWNLOAD "${ESPEAK_NG_URL}" "download/${ESPEAK_NG_ARCHIVE}") - endif() - - # Extract .zip or .tar.gz to lib/espeak-ng - file(ARCHIVE_EXTRACT INPUT "download/${ESPEAK_NG_ARCHIVE}" DESTINATION "${CMAKE_CURRENT_LIST_DIR}/lib") - endif() - endif() - - set(ESPEAK_NG_INCLUDE_DIRS "${ESPEAK_NG_DIR}/include") - set(ESPEAK_NG_LIBRARY_DIRS "${ESPEAK_NG_DIR}/lib") - set(ESPEAK_NG_LIBRARIES "espeak-ng") +if(NOT DEFINED ESPEAK_NG_DIR) + set(ESPEAK_NG_DIR "${CMAKE_CURRENT_BINARY_DIR}/espeak_ng_install") + + include(ExternalProject) + ExternalProject_Add( + espeak_ng_external + URL "https://github.com/rhasspy/espeak-ng/archive/refs/heads/master.zip" + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${ESPEAK_NG_DIR} + CMAKE_ARGS -DUSE_ASYNC:BOOL=OFF + CMAKE_ARGS -DBUILD_SHARED_LIBS:BOOL=ON + CMAKE_ARGS -DUSE_MBROLA:BOOL=OFF + CMAKE_ARGS -DUSE_LIBSONIC:BOOL=OFF + CMAKE_ARGS -DUSE_LIBPCAUDIO:BOOL=OFF + CMAKE_ARGS -DUSE_KLATT:BOOL=OFF + CMAKE_ARGS -DUSE_SPEECHPLAYER:BOOL=OFF + CMAKE_ARGS -DEXTRA_cmn:BOOL=ON + CMAKE_ARGS -DEXTRA_ru:BOOL=ON + ) + add_dependencies(piper_phonemize espeak_ng_external) endif() -# ---- Declare library ---- - -add_library( - piper_phonemize SHARED - src/phonemize.cpp - src/phoneme_ids.cpp - src/tashkeel.cpp - src/shared.cpp -) -set_target_properties(piper_phonemize PROPERTIES - VERSION ${PROJECT_VERSION} - SOVERSION ${PROJECT_VERSION_MAJOR}) +# ---- Declare library ---- target_include_directories( piper_phonemize PUBLIC "$" - ${ESPEAK_NG_INCLUDE_DIRS} + ${ESPEAK_NG_DIR}/include ${ONNXRUNTIME_ROOTDIR}/include ) target_link_directories( piper_phonemize PUBLIC - ${ESPEAK_NG_LIBRARY_DIRS} + ${ESPEAK_NG_DIR}/lib ${ONNXRUNTIME_ROOTDIR}/lib ) target_link_libraries( piper_phonemize - ${ESPEAK_NG_LIBRARIES} + espeak-ng onnxruntime ) -target_compile_options( - piper_phonemize PUBLIC - ${ESPEAK_NG_CFLAGS_OTHER} -) - target_compile_features(piper_phonemize PUBLIC cxx_std_17) # ---- Declare executable ---- @@ -174,21 +142,17 @@ target_compile_features(piper_phonemize_exe PUBLIC cxx_std_17) target_include_directories( piper_phonemize_exe PUBLIC "$" - ${ESPEAK_NG_INCLUDE_DIRS} + ${ESPEAK_NG_DIR}/include ) target_link_directories( piper_phonemize_exe PUBLIC - ${ESPEAK_NG_LIBRARY_DIRS} + ${ESPEAK_NG_DIR}/lib ) target_link_libraries(piper_phonemize_exe PUBLIC piper_phonemize - ${ESPEAK_NG_LIBRARIES}) - -target_compile_options( - piper_phonemize_exe PUBLIC - ${ESPEAK_NG_CFLAGS_OTHER} + espeak-ng ) # ---- Declare install targets ---- From f907a15103b9f2b61e4c146a16fde53dd85b5a0b Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Sat, 9 Sep 2023 11:24:05 -0500 Subject: [PATCH 17/21] Include dependencies during install --- CMakeLists.txt | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e37059f..44bd75a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,7 +36,7 @@ endif() # ---- onnxruntime --- # Look for onnxruntime files in /lib -if(NOT DEFINED ONNXRUNTIME_ROOTDIR) +if(NOT DEFINED ONNXRUNTIME_DIR) if(NOT DEFINED ONNXRUNTIME_VERSION) set(ONNXRUNTIME_VERSION "1.14.1") endif() @@ -68,9 +68,9 @@ if(NOT DEFINED ONNXRUNTIME_ROOTDIR) endif() set(ONNXRUNTIME_FILENAME "${ONNXRUNTIME_PREFIX}.${ONNXRUNTIME_EXT}") - set(ONNXRUNTIME_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/lib/${ONNXRUNTIME_PREFIX}") + set(ONNXRUNTIME_DIR "${CMAKE_CURRENT_LIST_DIR}/lib/${ONNXRUNTIME_PREFIX}") - if(NOT EXISTS "${ONNXRUNTIME_ROOTDIR}") + if(NOT EXISTS "${ONNXRUNTIME_DIR}") if(NOT EXISTS "download/${ONNXRUNTIME_FILENAME}") # Download onnxruntime release message("Downloading ${ONNXRUNTIME_URL}") @@ -112,13 +112,13 @@ target_include_directories( piper_phonemize PUBLIC "$" ${ESPEAK_NG_DIR}/include - ${ONNXRUNTIME_ROOTDIR}/include + ${ONNXRUNTIME_DIR}/include ) target_link_directories( piper_phonemize PUBLIC ${ESPEAK_NG_DIR}/lib - ${ONNXRUNTIME_ROOTDIR}/lib + ${ONNXRUNTIME_DIR}/lib ) target_link_libraries( @@ -156,6 +156,7 @@ target_link_libraries(piper_phonemize_exe PUBLIC ) # ---- Declare install targets ---- +include(GNUInstallDirs) install( TARGETS piper_phonemize @@ -171,3 +172,16 @@ install( install( TARGETS piper_phonemize_exe ARCHIVE DESTINATION ${CMAKE_INSTALL_BINDIR}) + +# Dependencies +install( + DIRECTORY ${ESPEAK_NG_DIR}/ + DESTINATION ${CMAKE_INSTALL_PREFIX}) + +install( + DIRECTORY ${ONNXRUNTIME_DIR}/include/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + +install( + DIRECTORY ${ONNXRUNTIME_DIR}/lib/ + DESTINATION ${CMAKE_INSTALL_LIBDIR}) From 0a24f783e859afadd31b12a84bc3710a703034bb Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Sat, 9 Sep 2023 13:27:26 -0500 Subject: [PATCH 18/21] Fix rpath --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 44bd75a..1e81a1b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,7 +29,7 @@ if(WIN32) else() # Linux flags - string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra -Wl,-rpath,'$$ORIGIN'") + string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra -Wl,-rpath,'$ORIGIN'") string(APPEND CMAKE_C_FLAGS " -Wall -Wextra") endif() @@ -156,6 +156,7 @@ target_link_libraries(piper_phonemize_exe PUBLIC ) # ---- Declare install targets ---- + include(GNUInstallDirs) install( From 39b62622dc5871e05d78399e97403a803fd0ae65 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Sat, 9 Sep 2023 14:15:19 -0500 Subject: [PATCH 19/21] Add test --- CMakeLists.txt | 35 ++++++++++++++++++++++++++++++++++- src/main.cpp | 2 +- src/phoneme_ids.cpp | 6 +++--- src/phoneme_ids.hpp | 7 ++++--- src/test.cpp | 7 ++++++- 5 files changed, 48 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e81a1b..9ce3ef4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,8 @@ add_library( set_target_properties(piper_phonemize PROPERTIES VERSION ${PROJECT_VERSION} - SOVERSION ${PROJECT_VERSION_MAJOR}) + SOVERSION ${PROJECT_VERSION_MAJOR} +) if(WIN32) # Force compiler to use UTF-8 for IPA constants @@ -155,6 +156,34 @@ target_link_libraries(piper_phonemize_exe PUBLIC espeak-ng ) +# ---- Declare test ---- + +include(CTest) +enable_testing() +add_executable(test_piper_phonemize src/test.cpp) +add_test( + NAME test_piper_phonemize + COMMAND test_piper_phonemize "${ESPEAK_NG_DIR}/share/espeak-ng-data" "${CMAKE_SOURCE_DIR}/etc/libtashkeel_model.ort" +) + +target_compile_features(test_piper_phonemize PUBLIC cxx_std_17) + +target_include_directories( + test_piper_phonemize PUBLIC + "$" + ${ESPEAK_NG_DIR}/include +) + +target_link_directories( + test_piper_phonemize PUBLIC + ${ESPEAK_NG_DIR}/lib +) + +target_link_libraries(test_piper_phonemize PUBLIC + piper_phonemize + espeak-ng +) + # ---- Declare install targets ---- include(GNUInstallDirs) @@ -174,6 +203,10 @@ install( TARGETS piper_phonemize_exe ARCHIVE DESTINATION ${CMAKE_INSTALL_BINDIR}) +install( + FILES ${CMAKE_SOURCE_DIR}/etc/libtashkeel_model.ort + TYPE DATA) + # Dependencies install( DIRECTORY ${ESPEAK_NG_DIR}/ diff --git a/src/main.cpp b/src/main.cpp index b8ef050..be1e233 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -24,7 +24,7 @@ using json = nlohmann::json; enum PhonemeType { eSpeakPhonemes, TextPhonemes }; struct RunConfig { - std::string language; + std::string language = ""; PhonemeType phonemeType = eSpeakPhonemes; std::optional eSpeakDataPath; std::optional tashkeelModelPath; diff --git a/src/phoneme_ids.cpp b/src/phoneme_ids.cpp index 760b04a..3ba586c 100644 --- a/src/phoneme_ids.cpp +++ b/src/phoneme_ids.cpp @@ -6,8 +6,8 @@ namespace piper { -const size_t MAX_PHONEMES = 256; -PhonemeIdMap DEFAULT_PHONEME_ID_MAP = { +PIPERPHONEMIZE_EXPORT const size_t MAX_PHONEMES = 256; +PIPERPHONEMIZE_EXPORT PhonemeIdMap DEFAULT_PHONEME_ID_MAP = { {U'_', {0}}, {U'^', {1}}, {U'$', {2}}, @@ -179,7 +179,7 @@ PhonemeIdMap DEFAULT_PHONEME_ID_MAP = { }; // language -> phoneme -> [id, ...] -std::map DEFAULT_ALPHABET = { +PIPERPHONEMIZE_EXPORT std::map DEFAULT_ALPHABET = { // Ukrainian {"uk", { diff --git a/src/phoneme_ids.hpp b/src/phoneme_ids.hpp index 0fe3cea..1ec3c65 100644 --- a/src/phoneme_ids.hpp +++ b/src/phoneme_ids.hpp @@ -32,9 +32,10 @@ struct PhonemeIdConfig { std::shared_ptr phonemeIdMap; }; -extern const size_t MAX_PHONEMES; -extern PhonemeIdMap DEFAULT_PHONEME_ID_MAP; -extern std::map DEFAULT_ALPHABET; +extern PIPERPHONEMIZE_EXPORT const size_t MAX_PHONEMES; +extern PIPERPHONEMIZE_EXPORT PhonemeIdMap DEFAULT_PHONEME_ID_MAP; +extern PIPERPHONEMIZE_EXPORT std::map + DEFAULT_ALPHABET; PIPERPHONEMIZE_EXPORT void phonemes_to_ids(const std::vector &phonemes, PhonemeIdConfig &config, diff --git a/src/test.cpp b/src/test.cpp index f580b60..b7619a1 100644 --- a/src/test.cpp +++ b/src/test.cpp @@ -51,6 +51,11 @@ int main(int argc, char *argv[]) { return 1; } + if (argc < 3) { + std::cerr << "Need tashkeel model path" << std::endl; + return 1; + } + int result = espeak_Initialize(AUDIO_OUTPUT_SYNCHRONOUS, 0, argv[1], 0); if (result < 0) { std::cerr << "Failed to initialize eSpeak" << std::endl; @@ -163,7 +168,7 @@ int main(int argc, char *argv[]) { // Test Arabic with libtashkeel (https://github.com/mush42/libtashkeel) tashkeel::State tashkeelState; - tashkeel::tashkeel_load("etc/libtashkeel_model.ort", tashkeelState); + tashkeel::tashkeel_load(argv[2], tashkeelState); std::string expectedText = "مَرْحَبًا"; std::string actualText = tashkeel::tashkeel_run("مرحبا", tashkeelState); From 9b18c025915e3387bd593ef53d55984cef411430 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Sat, 9 Sep 2023 16:16:24 -0500 Subject: [PATCH 20/21] Adjusting for Windows --- CMakeLists.txt | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ce3ef4..e14b798 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,17 @@ project( LANGUAGES CXX ) +if(MSVC) + # Force compiler to use UTF-8 for IPA constants + add_compile_options("$<$:/utf-8>") + add_compile_options("$<$:/utf-8>") + +elseif(NOT APPLE) + # Linux flags + string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra -Wl,-rpath,'$ORIGIN'") + string(APPEND CMAKE_C_FLAGS " -Wall -Wextra") +endif() + add_library( piper_phonemize SHARED src/phonemize.cpp @@ -23,17 +34,6 @@ set_target_properties(piper_phonemize PROPERTIES SOVERSION ${PROJECT_VERSION_MAJOR} ) -if(WIN32) - # Force compiler to use UTF-8 for IPA constants - add_compile_options("$<$:/utf-8>") - add_compile_options("$<$:/utf-8>") - -else() - # Linux flags - string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra -Wl,-rpath,'$ORIGIN'") - string(APPEND CMAKE_C_FLAGS " -Wall -Wextra") -endif() - # ---- onnxruntime --- # Look for onnxruntime files in /lib @@ -86,11 +86,12 @@ endif() # ---- espeak-ng --- if(NOT DEFINED ESPEAK_NG_DIR) - set(ESPEAK_NG_DIR "${CMAKE_CURRENT_BINARY_DIR}/espeak_ng_install") + set(ESPEAK_NG_DIR "${CMAKE_CURRENT_BINARY_DIR}/ei") include(ExternalProject) ExternalProject_Add( espeak_ng_external + PREFIX "${CMAKE_CURRENT_BINARY_DIR}/e" URL "https://github.com/rhasspy/espeak-ng/archive/refs/heads/master.zip" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${ESPEAK_NG_DIR} CMAKE_ARGS -DUSE_ASYNC:BOOL=OFF @@ -160,7 +161,7 @@ target_link_libraries(piper_phonemize_exe PUBLIC include(CTest) enable_testing() -add_executable(test_piper_phonemize src/test.cpp) +add_executable(test_piper_phonemize src/test.cpp src/phoneme_ids.cpp) add_test( NAME test_piper_phonemize COMMAND test_piper_phonemize "${ESPEAK_NG_DIR}/share/espeak-ng-data" "${CMAKE_SOURCE_DIR}/etc/libtashkeel_model.ort" From 64c15536ac9b5251476922b0a3771f29b2d68a57 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Sat, 9 Sep 2023 21:00:27 -0500 Subject: [PATCH 21/21] Move extern to static --- .dockerignore | 11 --- Dockerfile | 125 ---------------------------- Makefile | 50 ++---------- espeak-ng | 1 - src/phoneme_ids.cpp | 192 ------------------------------------------- src/phoneme_ids.hpp | 195 +++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 199 insertions(+), 375 deletions(-) delete mode 100644 .dockerignore delete mode 100644 Dockerfile delete mode 160000 espeak-ng diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 917da8d..0000000 --- a/.dockerignore +++ /dev/null @@ -1,11 +0,0 @@ -* -!CMakeLists.txt -!Makefile -!src/ -!setup.py -!pyproject.toml -!piper_phonemize/ -!LICENSE.md -!README.md -!MANIFEST.in -!etc/libtashkeel_model.ort diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 9724023..0000000 --- a/Dockerfile +++ /dev/null @@ -1,125 +0,0 @@ -FROM quay.io/pypa/manylinux_2_28_x86_64 as build-amd64 -ARG ONNXRUNTIME_VERSION='1.14.1' -ENV ONNXRUNTIME_URL="https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz" - -FROM quay.io/pypa/manylinux_2_28_aarch64 as build-arm64 -ARG ONNXRUNTIME_VERSION='1.14.1' -ENV ONNXRUNTIME_URL="https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-aarch64-${ONNXRUNTIME_VERSION}.tgz" - -FROM debian:bullseye as build-armv7 -ARG ONNXRUNTIME_VERSION='1.14.1' -ENV ONNXRUNTIME_URL="https://github.com/synesthesiam/prebuilt-apps/releases/download/v1.0/onnxruntime-linux-arm32-${ONNXRUNTIME_VERSION}.tgz" - -ENV LANG C.UTF-8 -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && \ - apt-get install --yes \ - build-essential cmake curl ca-certificates autoconf automake libtool pkg-config - -# ----------------------------------------------------------------------------- - -ARG TARGETARCH -ARG TARGETVARIANT -FROM build-${TARGETARCH}${TARGETVARIANT} as build -ARG TARGETARCH -ARG TARGETVARIANT - -ENV LANG C.UTF-8 -ENV DEBIAN_FRONTEND=noninteractive - -WORKDIR /build - -RUN mkdir -p "lib/Linux-$(uname -m)" - -# Download and extract onnxruntime -RUN curl -L "${ONNXRUNTIME_URL}" | \ - tar -C "lib/Linux-$(uname -m)" -xzvf - && \ - mv "lib/Linux-$(uname -m)"/onnxruntime-* \ - "lib/Linux-$(uname -m)/onnxruntime" - -# Build minimal version of espeak-ng -RUN curl -L "https://github.com/rhasspy/espeak-ng/archive/refs/heads/master.tar.gz" | \ - tar -xzf - - -RUN cd espeak-ng-master && \ - export CFLAGS='-D_FILE_OFFSET_BITS=64' && \ - ./autogen.sh && \ - ./configure \ - --without-pcaudiolib \ - --without-klatt \ - --without-speechplayer \ - --without-mbrola \ - --without-sonic \ - --with-extdict-cmn \ - --with-extdict-ru \ - --prefix=/usr && \ - make -j8 src/espeak-ng src/speak-ng && \ - make && \ - make install - -# For future, not currently working -# cmake -Bbuild \ -# -DCMAKE_INSTALL_PREFIX=/usr \ -# -DCMAKE_BUILD_TYPE=Release \ -# -DBUILD_SHARED_LIBS=ON \ -# -DUSE_MBROLA=OFF \ -# -DUSE_LIBSONIC=OFF \ -# -DUSE_LIBPCAUDIO=OFF \ -# -DUSE_KLATT=OFF \ -# -DUSE_SPEECHPLAYER=OFF \ -# -DUSE_ASYNC=OFF \ -# -DEXTRA_cmn=ON \ -# -DEXTRA_ru=ON && \ - -# Build libpiper_phonemize.so -COPY etc/libtashkeel_model.ort ./etc/ -COPY CMakeLists.txt Makefile ./ -COPY src/ ./src/ - -# Sanity check -RUN make test - -# Build libpiper_phonemize.so -RUN mkdir build && \ - cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release .. && \ - make - -# Package libpiper_phonemize.so and piper_phonemize -RUN mkdir -p /dist/lib && \ - cd /dist && \ - cp /build/build/libpiper_phonemize.so* /build/build/piper_phonemize ./lib/ && \ - find /usr -name 'libespeak-ng*.so*' -exec cp -a {} ./lib/ \; && \ - find /usr -type d -name 'espeak-ng-data' -exec cp -R {} ./lib/ \; && \ - mkdir -p ./include && \ - cp -R /usr/include/espeak-ng ./include/ && \ - cp /build/src/phonemize.hpp /build/src/phoneme_ids.hpp /build/src/tashkeel.hpp ./include/ && \ - cp -a "/build/lib/Linux-$(uname -m)/onnxruntime/lib"/libonnxruntime*.so* ./lib/ && \ - cp -R "/build/lib/Linux-$(uname -m)/onnxruntime/include"/* ./include/ && \ - cp -R /build/etc ./ && \ - tar -czf libpiper_phonemize.tar.gz * - -# Build piper_phonemize Python package -COPY setup.py pyproject.toml MANIFEST.in README.md LICENSE.md ./ -COPY piper_phonemize/ ./piper_phonemize/ -RUN find /usr -type d -name 'espeak-ng-data' -exec cp -R {} ./piper_phonemize/ \; && \ - cp /build/etc/libtashkeel_model.ort ./piper_phonemize/ - -RUN mkdir -p wheelhouse -RUN if [ "$(which auditwheel)" ]; then \ - /opt/python/cp39-cp39/bin/pip wheel . && \ - /opt/python/cp310-cp310/bin/pip wheel . && \ - /opt/python/cp311-cp311/bin/pip wheel . && \ - cp -a "/build/lib/Linux-$(uname -m)/onnxruntime/lib"/libonnxruntime*.so* /usr/lib/ && \ - auditwheel repair *.whl; \ - fi - -# ----------------------------------------------------------------------------- - -FROM scratch -ARG TARGETARCH -ARG TARGETVARIANT - -COPY --from=build /dist/libpiper_phonemize.tar.gz ./libpiper_phonemize-${TARGETARCH}${TARGETVARIANT}.tar.gz -COPY --from=build /build/wheelhouse/ ./ diff --git a/Makefile b/Makefile index 6ef9d5b..79e96c3 100644 --- a/Makefile +++ b/Makefile @@ -1,44 +1,10 @@ -.PHONY: release release-windows test python python-test docker docker-windows +.PHONY: clean -LIB_DIR := lib/Linux-$(shell uname -m) -DOCKER_PLATFORM ?= linux/amd64,linux/arm64,linux/arm/v7 -VENV ?= .venv +all: + cmake -Bbuild -DCMAKE_INSTALL_PREFIX=install + cmake --build build --config Release + cd build && ctest --config Release + cmake --install install -release: - mkdir -p build - cd build && cmake .. -DCMAKE_BUILD_TYPE=Release && make - cp -a espeak-ng/build/lib/libespeak*.so* build/ - cp -R espeak-ng/build/share/espeak-ng-data build/ - cp -a $(LIB_DIR)/onnxruntime/lib/libonnxruntime*.so* build/ - cp etc/libtashkeel_model.ort build/ - -release-windows: - mkdir -p build - cd build && cmake .. -DCMAKE_BUILD_TYPE=Release && make - cp -a espeak-ng/build_windows/lib/libespeak*.dll build/ - cp -R espeak-ng/build/share/espeak-ng-data build/ - cp -a $(LIB_DIR)/onnxruntime/lib/libonnxruntime*.dll build/ - cp etc/libtashkeel_model.ort build/ - -test: - g++ -Wall -o test -Iespeak-ng/build/include -Lespeak-ng/build/lib -I$(LIB_DIR)/onnxruntime/include -L$(LIB_DIR)/onnxruntime/lib -Isrc -std=c++17 src/test.cpp src/phonemize.cpp src/phoneme_ids.cpp src/tashkeel.cpp src/shared.cpp -lespeak-ng -lonnxruntime - LD_LIBRARY_PATH="espeak-ng/build/lib:$(LIB_DIR)/onnxruntime/lib:${LD_LIBRARY_PATH}" ./test 'espeak-ng/build/share/espeak-ng-data' - -python: - cp -R espeak-ng/build/share/espeak-ng-data piper_phonemize/ - cp etc/libtashkeel_model.ort piper_phonemize/ - LD_LIBRARY_PATH="espeak-ng/build/lib:${LD_LIBRARY_PATH}" "$(VENV)/bin/pip3" install -e . - -python-test: - LD_LIBRARY_PATH="espeak-ng/build/lib:$(LIB_DIR)/onnxruntime/lib:${LD_LIBRARY_PATH}" "$(VENV)/bin/python3" src/python_test.py - -python-wheel: - cp -R espeak-ng/build/share/espeak-ng-data piper_phonemize/ - cp etc/libtashkeel_model.ort piper_phonemize/ - LD_LIBRARY_PATH="espeak-ng/build/lib:${LD_LIBRARY_PATH}" "$(VENV)/bin/python3" setup.py bdist_wheel - -docker: - docker buildx build . --platform "$(DOCKER_PLATFORM)" --output 'type=local,dest=dist' - -docker-windows: - docker buildx build . -f Dockerfile.windows --output 'type=local,dest=dist/windows_amd64' +clean: + rm -rf build install diff --git a/espeak-ng b/espeak-ng deleted file mode 160000 index 6fd1834..0000000 --- a/espeak-ng +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6fd1834efbb19c28f0bad5328b409ce00582fee1 diff --git a/src/phoneme_ids.cpp b/src/phoneme_ids.cpp index 3ba586c..b6a0d40 100644 --- a/src/phoneme_ids.cpp +++ b/src/phoneme_ids.cpp @@ -6,198 +6,6 @@ namespace piper { -PIPERPHONEMIZE_EXPORT const size_t MAX_PHONEMES = 256; -PIPERPHONEMIZE_EXPORT PhonemeIdMap DEFAULT_PHONEME_ID_MAP = { - {U'_', {0}}, - {U'^', {1}}, - {U'$', {2}}, - {U' ', {3}}, - {U'!', {4}}, - {U'\'', {5}}, - {U'(', {6}}, - {U')', {7}}, - {U',', {8}}, - {U'-', {9}}, - {U'.', {10}}, - {U':', {11}}, - {U';', {12}}, - {U'?', {13}}, - {U'a', {14}}, - {U'b', {15}}, - {U'c', {16}}, - {U'd', {17}}, - {U'e', {18}}, - {U'f', {19}}, - {U'h', {20}}, - {U'i', {21}}, - {U'j', {22}}, - {U'k', {23}}, - {U'l', {24}}, - {U'm', {25}}, - {U'n', {26}}, - {U'o', {27}}, - {U'p', {28}}, - {U'q', {29}}, - {U'r', {30}}, - {U's', {31}}, - {U't', {32}}, - {U'u', {33}}, - {U'v', {34}}, - {U'w', {35}}, - {U'x', {36}}, - {U'y', {37}}, - {U'z', {38}}, - {U'æ', {39}}, - {U'ç', {40}}, - {U'ð', {41}}, - {U'ø', {42}}, - {U'ħ', {43}}, - {U'ŋ', {44}}, - {U'œ', {45}}, - {U'ǀ', {46}}, - {U'ǁ', {47}}, - {U'ǂ', {48}}, - {U'ǃ', {49}}, - {U'ɐ', {50}}, - {U'ɑ', {51}}, - {U'ɒ', {52}}, - {U'ɓ', {53}}, - {U'ɔ', {54}}, - {U'ɕ', {55}}, - {U'ɖ', {56}}, - {U'ɗ', {57}}, - {U'ɘ', {58}}, - {U'ə', {59}}, - {U'ɚ', {60}}, - {U'ɛ', {61}}, - {U'ɜ', {62}}, - {U'ɞ', {63}}, - {U'ɟ', {64}}, - {U'ɠ', {65}}, - {U'ɡ', {66}}, - {U'ɢ', {67}}, - {U'ɣ', {68}}, - {U'ɤ', {69}}, - {U'ɥ', {70}}, - {U'ɦ', {71}}, - {U'ɧ', {72}}, - {U'ɨ', {73}}, - {U'ɪ', {74}}, - {U'ɫ', {75}}, - {U'ɬ', {76}}, - {U'ɭ', {77}}, - {U'ɮ', {78}}, - {U'ɯ', {79}}, - {U'ɰ', {80}}, - {U'ɱ', {81}}, - {U'ɲ', {82}}, - {U'ɳ', {83}}, - {U'ɴ', {84}}, - {U'ɵ', {85}}, - {U'ɶ', {86}}, - {U'ɸ', {87}}, - {U'ɹ', {88}}, - {U'ɺ', {89}}, - {U'ɻ', {90}}, - {U'ɽ', {91}}, - {U'ɾ', {92}}, - {U'ʀ', {93}}, - {U'ʁ', {94}}, - {U'ʂ', {95}}, - {U'ʃ', {96}}, - {U'ʄ', {97}}, - {U'ʈ', {98}}, - {U'ʉ', {99}}, - {U'ʊ', {100}}, - {U'ʋ', {101}}, - {U'ʌ', {102}}, - {U'ʍ', {103}}, - {U'ʎ', {104}}, - {U'ʏ', {105}}, - {U'ʐ', {106}}, - {U'ʑ', {107}}, - {U'ʒ', {108}}, - {U'ʔ', {109}}, - {U'ʕ', {110}}, - {U'ʘ', {111}}, - {U'ʙ', {112}}, - {U'ʛ', {113}}, - {U'ʜ', {114}}, - {U'ʝ', {115}}, - {U'ʟ', {116}}, - {U'ʡ', {117}}, - {U'ʢ', {118}}, - {U'ʲ', {119}}, - {U'ˈ', {120}}, - {U'ˌ', {121}}, - {U'ː', {122}}, - {U'ˑ', {123}}, - {U'˞', {124}}, - {U'β', {125}}, - {U'θ', {126}}, - {U'χ', {127}}, - {U'ᵻ', {128}}, - {U'ⱱ', {129}}, - - // tones - {U'0', {130}}, - {U'1', {131}}, - {U'2', {132}}, - {U'3', {133}}, - {U'4', {134}}, - {U'5', {135}}, - {U'6', {136}}, - {U'7', {137}}, - {U'8', {138}}, - {U'9', {139}}, - {U'\u0327', {140}}, // combining cedilla - {U'\u0303', {141}}, // combining tilde - {U'\u032a', {142}}, // combining bridge below - {U'\u032f', {143}}, // combining inverted breve below - {U'\u0329', {144}}, // combining vertical line below - {U'ʰ', {145}}, - {U'ˤ', {146}}, - {U'ε', {147}}, - {U'↓', {148}}, - {U'#', {149}}, // Icelandic - {U'\"', {150}}, // Russian - - {U'↑', {151}}, - - // Basque - {U'\u033a', {152}}, - {U'\u033b', {153}}, - - // Luxembourgish - {U'g', {154}}, - {U'ʦ', {155}}, - {U'X', {156}}, - - // Czech - {U'\u031d', {157}}, - {U'\u030a', {158}}, -}; - -// language -> phoneme -> [id, ...] -PIPERPHONEMIZE_EXPORT std::map DEFAULT_ALPHABET = { - // Ukrainian - {"uk", - { - {U'_', {0}}, {U'^', {1}}, {U'$', {2}}, {U' ', {3}}, - {U'!', {4}}, {U'\'', {5}}, {U',', {6}}, {U'-', {7}}, - {U'.', {8}}, {U':', {9}}, {U';', {10}}, {U'?', {11}}, - {U'а', {12}}, {U'б', {13}}, {U'в', {14}}, {U'г', {15}}, - {U'ґ', {16}}, {U'д', {17}}, {U'е', {18}}, {U'є', {19}}, - {U'ж', {20}}, {U'з', {21}}, {U'и', {22}}, {U'і', {23}}, - {U'ї', {24}}, {U'й', {25}}, {U'к', {26}}, {U'л', {27}}, - {U'м', {28}}, {U'н', {29}}, {U'о', {30}}, {U'п', {31}}, - {U'р', {32}}, {U'с', {33}}, {U'т', {34}}, {U'у', {35}}, - {U'ф', {36}}, {U'х', {37}}, {U'ц', {38}}, {U'ч', {39}}, - {U'ш', {40}}, {U'щ', {41}}, {U'ь', {42}}, {U'ю', {43}}, - {U'я', {44}}, {U'\u0301', {45}}, {U'\u0306', {46}}, {U'\u0308', {47}}, - {U'—', {48}}, - }}}; - PIPERPHONEMIZE_EXPORT void phonemes_to_ids(const std::vector &phonemes, PhonemeIdConfig &config, std::vector &phonemeIds, diff --git a/src/phoneme_ids.hpp b/src/phoneme_ids.hpp index 1ec3c65..8978df0 100644 --- a/src/phoneme_ids.hpp +++ b/src/phoneme_ids.hpp @@ -32,10 +32,197 @@ struct PhonemeIdConfig { std::shared_ptr phonemeIdMap; }; -extern PIPERPHONEMIZE_EXPORT const size_t MAX_PHONEMES; -extern PIPERPHONEMIZE_EXPORT PhonemeIdMap DEFAULT_PHONEME_ID_MAP; -extern PIPERPHONEMIZE_EXPORT std::map - DEFAULT_ALPHABET; +static const size_t MAX_PHONEMES = 256; +static PhonemeIdMap DEFAULT_PHONEME_ID_MAP = { + {U'_', {0}}, + {U'^', {1}}, + {U'$', {2}}, + {U' ', {3}}, + {U'!', {4}}, + {U'\'', {5}}, + {U'(', {6}}, + {U')', {7}}, + {U',', {8}}, + {U'-', {9}}, + {U'.', {10}}, + {U':', {11}}, + {U';', {12}}, + {U'?', {13}}, + {U'a', {14}}, + {U'b', {15}}, + {U'c', {16}}, + {U'd', {17}}, + {U'e', {18}}, + {U'f', {19}}, + {U'h', {20}}, + {U'i', {21}}, + {U'j', {22}}, + {U'k', {23}}, + {U'l', {24}}, + {U'm', {25}}, + {U'n', {26}}, + {U'o', {27}}, + {U'p', {28}}, + {U'q', {29}}, + {U'r', {30}}, + {U's', {31}}, + {U't', {32}}, + {U'u', {33}}, + {U'v', {34}}, + {U'w', {35}}, + {U'x', {36}}, + {U'y', {37}}, + {U'z', {38}}, + {U'æ', {39}}, + {U'ç', {40}}, + {U'ð', {41}}, + {U'ø', {42}}, + {U'ħ', {43}}, + {U'ŋ', {44}}, + {U'œ', {45}}, + {U'ǀ', {46}}, + {U'ǁ', {47}}, + {U'ǂ', {48}}, + {U'ǃ', {49}}, + {U'ɐ', {50}}, + {U'ɑ', {51}}, + {U'ɒ', {52}}, + {U'ɓ', {53}}, + {U'ɔ', {54}}, + {U'ɕ', {55}}, + {U'ɖ', {56}}, + {U'ɗ', {57}}, + {U'ɘ', {58}}, + {U'ə', {59}}, + {U'ɚ', {60}}, + {U'ɛ', {61}}, + {U'ɜ', {62}}, + {U'ɞ', {63}}, + {U'ɟ', {64}}, + {U'ɠ', {65}}, + {U'ɡ', {66}}, + {U'ɢ', {67}}, + {U'ɣ', {68}}, + {U'ɤ', {69}}, + {U'ɥ', {70}}, + {U'ɦ', {71}}, + {U'ɧ', {72}}, + {U'ɨ', {73}}, + {U'ɪ', {74}}, + {U'ɫ', {75}}, + {U'ɬ', {76}}, + {U'ɭ', {77}}, + {U'ɮ', {78}}, + {U'ɯ', {79}}, + {U'ɰ', {80}}, + {U'ɱ', {81}}, + {U'ɲ', {82}}, + {U'ɳ', {83}}, + {U'ɴ', {84}}, + {U'ɵ', {85}}, + {U'ɶ', {86}}, + {U'ɸ', {87}}, + {U'ɹ', {88}}, + {U'ɺ', {89}}, + {U'ɻ', {90}}, + {U'ɽ', {91}}, + {U'ɾ', {92}}, + {U'ʀ', {93}}, + {U'ʁ', {94}}, + {U'ʂ', {95}}, + {U'ʃ', {96}}, + {U'ʄ', {97}}, + {U'ʈ', {98}}, + {U'ʉ', {99}}, + {U'ʊ', {100}}, + {U'ʋ', {101}}, + {U'ʌ', {102}}, + {U'ʍ', {103}}, + {U'ʎ', {104}}, + {U'ʏ', {105}}, + {U'ʐ', {106}}, + {U'ʑ', {107}}, + {U'ʒ', {108}}, + {U'ʔ', {109}}, + {U'ʕ', {110}}, + {U'ʘ', {111}}, + {U'ʙ', {112}}, + {U'ʛ', {113}}, + {U'ʜ', {114}}, + {U'ʝ', {115}}, + {U'ʟ', {116}}, + {U'ʡ', {117}}, + {U'ʢ', {118}}, + {U'ʲ', {119}}, + {U'ˈ', {120}}, + {U'ˌ', {121}}, + {U'ː', {122}}, + {U'ˑ', {123}}, + {U'˞', {124}}, + {U'β', {125}}, + {U'θ', {126}}, + {U'χ', {127}}, + {U'ᵻ', {128}}, + {U'ⱱ', {129}}, + + // tones + {U'0', {130}}, + {U'1', {131}}, + {U'2', {132}}, + {U'3', {133}}, + {U'4', {134}}, + {U'5', {135}}, + {U'6', {136}}, + {U'7', {137}}, + {U'8', {138}}, + {U'9', {139}}, + {U'\u0327', {140}}, // combining cedilla + {U'\u0303', {141}}, // combining tilde + {U'\u032a', {142}}, // combining bridge below + {U'\u032f', {143}}, // combining inverted breve below + {U'\u0329', {144}}, // combining vertical line below + {U'ʰ', {145}}, + {U'ˤ', {146}}, + {U'ε', {147}}, + {U'↓', {148}}, + {U'#', {149}}, // Icelandic + {U'\"', {150}}, // Russian + + {U'↑', {151}}, + + // Basque + {U'\u033a', {152}}, + {U'\u033b', {153}}, + + // Luxembourgish + {U'g', {154}}, + {U'ʦ', {155}}, + {U'X', {156}}, + + // Czech + {U'\u031d', {157}}, + {U'\u030a', {158}}, +}; + +// language -> phoneme -> [id, ...] +static std::map DEFAULT_ALPHABET = { + // Ukrainian + {"uk", + { + {U'_', {0}}, {U'^', {1}}, {U'$', {2}}, {U' ', {3}}, + {U'!', {4}}, {U'\'', {5}}, {U',', {6}}, {U'-', {7}}, + {U'.', {8}}, {U':', {9}}, {U';', {10}}, {U'?', {11}}, + {U'а', {12}}, {U'б', {13}}, {U'в', {14}}, {U'г', {15}}, + {U'ґ', {16}}, {U'д', {17}}, {U'е', {18}}, {U'є', {19}}, + {U'ж', {20}}, {U'з', {21}}, {U'и', {22}}, {U'і', {23}}, + {U'ї', {24}}, {U'й', {25}}, {U'к', {26}}, {U'л', {27}}, + {U'м', {28}}, {U'н', {29}}, {U'о', {30}}, {U'п', {31}}, + {U'р', {32}}, {U'с', {33}}, {U'т', {34}}, {U'у', {35}}, + {U'ф', {36}}, {U'х', {37}}, {U'ц', {38}}, {U'ч', {39}}, + {U'ш', {40}}, {U'щ', {41}}, {U'ь', {42}}, {U'ю', {43}}, + {U'я', {44}}, {U'\u0301', {45}}, {U'\u0306', {46}}, {U'\u0308', {47}}, + {U'—', {48}}, + }}}; PIPERPHONEMIZE_EXPORT void phonemes_to_ids(const std::vector &phonemes, PhonemeIdConfig &config,