From 4e487f4429e6748e78fbdd313d35f543e3f30542 Mon Sep 17 00:00:00 2001 From: johanbeytell1 <107624055+johanbeytell1@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:43:14 +0000 Subject: [PATCH 1/2] Prepare ValX 0.2.7 and enable PyPI Trusted Publishing Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/build_and_publish.yml | 95 +++-- .github/workflows/test.yml | 33 ++ .gitignore | 5 +- CONTRIBUTING.md | 42 ++- README.md | 68 ++-- pyproject.toml | 37 ++ requirements.txt | 2 +- setup.py | 33 -- test.py | 401 +++++++++------------- valx/__init__.py | 438 ++++++++++-------------- 10 files changed, 547 insertions(+), 607 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml index 4a9ec6b..1601d52 100644 --- a/.github/workflows/build_and_publish.yml +++ b/.github/workflows/build_and_publish.yml @@ -1,34 +1,77 @@ -name: Publish Python 🐍 distributions 📦 to PyPI +name: Publish to PyPI on: push: tags: - - '*' + - '[0-9]+.[0-9]+.[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+' + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false jobs: - build-n-publish: - name: Build and publish Python 🐍 distributions 📦 to PyPI + build: + name: Build and test release distributions + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + artifact-id: ${{ steps.upload.outputs.artifact-id }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + - name: Check that the release tag matches the package version + run: | + python - <<'PY' + import os + import tomllib + + with open('pyproject.toml', 'rb') as config: + version = tomllib.load(config)['project']['version'] + tag = os.environ['GITHUB_REF_NAME'] + if tag.removeprefix('v') != version: + raise SystemExit(f'Release tag {tag!r} does not match package version {version!r}.') + PY + - name: Build wheel and source distribution + run: | + python -m pip install build twine + python -m build + python -m twine check --strict dist/* + - name: Test the installed wheel + run: | + python -m pip install dist/*.whl + python -I test.py + - name: Upload checked distributions + id: upload + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: python-distributions + path: dist/ + if-no-files-found: error + + publish: + name: Publish with Trusted Publishing and attestations + if: github.repository == 'Infinitode/ValX' + needs: build runs-on: ubuntu-latest + permissions: + id-token: write # Request the short-lived PyPI credential and sign the attestations. + # The PyPI Trusted Publisher is registered with its Environment field blank. steps: - - uses: actions/checkout@master - - name: Set up Python 3.12 - uses: actions/setup-python@v3 - with: - python-version: '3.12' - - name: Install pypa/setuptools - run: >- - python -m - pip install setuptools wheel - - name: Extract tag name - id: tag - run: echo ::set-output name=TAG_NAME::$(echo $GITHUB_REF | cut -d / -f 3) - - name: Update version in setup.py - run: >- - sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" setup.py - - name: Build a binary wheel - run: >- - python setup.py sdist bdist_wheel - - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + - name: Download the checked distributions from this run + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + artifact-ids: ${{ needs.build.outputs.artifact-id }} + path: dist/ + digest-mismatch: error + - name: Upload wheel and sdist with PEP 740 publish attestations + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + attestations: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..f82b7a9 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,33 @@ +name: Test ValX + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read # Check out the code to test. + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Test on Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install ValX + run: python -m pip install . + - name: Test the installed package + run: python -I test.py diff --git a/.gitignore b/.gitignore index db83fe1..57e532c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ -__pycache__ +__pycache__/ *.pyc dist/ build/ +*.egg-info/ env/ -.venv/ \ No newline at end of file +.venv/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 91a0fe2..bf0e596 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,13 +8,12 @@ Thank you for considering contributing to **ValX**! Contributions, whether they Create a fork of this repository to your GitHub account and clone it locally. 2. **Set Up the Environment** - Ensure you have Python 3.6+ installed. You should also install compatible versions of the necessary dependencies for the project. + Use Python 3.9 or newer. In a virtual environment, install ValX for development and run its tests: -3. **Run Tests** - Before making changes, run tests to confirm that everything works. - -> [!TIP] -> We sometimes include testing scripts in our projects for easy testing. + ```bash + python -m pip install -r requirements.txt + python -m unittest discover -v + ``` ## How to Contribute @@ -25,7 +24,7 @@ Thank you for considering contributing to **ValX**! Contributions, whether they 2. **Make Changes** - Use clear and descriptive commit messages. - - (Optional) Write tests to show new functionality. + - Add regression tests for bug fixes and tests for new behavior. - Ensure the code is **readable**. You can learn more about Python code readability here: https://peps.python.org/pep-0008/. 3. **Submit a Pull Request** @@ -33,6 +32,33 @@ Thank you for considering contributing to **ValX**! Contributions, whether they - Submit a pull request with a detailed explanation of what you've changed or added. - Ensure your PR passes all automated tests and adheres to the contribution guidelines. +## Releasing to PyPI + +Register a GitHub Actions Trusted Publisher under **Publishing** in [ValX's PyPI project settings](https://pypi.org/manage/projects/): + +| PyPI field | Value | +| --- | --- | +| Owner | `Infinitode` | +| Repository name | `ValX` | +| Workflow name | `build_and_publish.yml` | +| Environment name | **Leave blank** (do not enter `None` or `pypi`) | + +The publish job deliberately has no GitHub environment and uses OIDC rather than a `PYPI_API_TOKEN`. Limit who can create or change release tags with a GitHub tag ruleset, since there is no environment approval gate. + +1. Set the version in `pyproject.toml`, update the README change notes, and merge the reviewed changes. +2. Tag the merged commit and push the tag, for example `git tag v0.2.7` followed by `git push origin v0.2.7`. Use a final `X.Y.Z` or `vX.Y.Z` tag matching the package version. +3. Once the workflow succeeds, open the release's **Security** tab on PyPI. Confirm there is a publish attestation for both the wheel and the source distribution from `Infinitode/ValX` and `build_and_publish.yml`. Avoid separate manual uploads, which would not inherit those attestations. + +To verify the signatures against this repository, use `pypi-attestations` after the release is live (substitute the actual version): + +```bash +python -m pip install pypi-attestations +pypi-attestations verify pypi --repository https://github.com/Infinitode/ValX pypi:valx-0.2.7-py3-none-any.whl +pypi-attestations verify pypi --repository https://github.com/Infinitode/ValX pypi:valx-0.2.7.tar.gz +``` + +After the first verified OIDC release, remove the unused `PYPI_API_TOKEN` secret from GitHub and revoke the old token on PyPI if nothing else uses it. + ## Community Guidelines To maintain a positive and collaborative community, we ask that all contributors adhere to the following principles: @@ -46,4 +72,4 @@ To maintain a positive and collaborative community, we ask that all contributors 3. **Follow Licensing Requirements** Ensure that any derivative works comply with the [license](https://github.com/infinitode/ValX/blob/main/LICENSE). -By contributing, you agree to abide by these guidelines and this project's [license](https://github.com/infinitode/ValX/blob/main/LICENSE). Thank you for helping make **ValX** better! \ No newline at end of file +By contributing, you agree to abide by these guidelines and this project's [license](https://github.com/infinitode/ValX/blob/main/LICENSE). Thank you for helping make **ValX** better! diff --git a/README.md b/README.md index 23868b3..ff71a94 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,20 @@ # ValX -![Python Version](https://img.shields.io/badge/python-3.12-blue.svg) +![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue.svg) [![Code Size](https://img.shields.io/github/languages/code-size/infinitode/valx)](https://github.com/infinitode/valx) ![Downloads](https://pepy.tech/badge/valx) ![License Compliance](https://img.shields.io/badge/license-compliance-brightgreen.svg) ![PyPI Version](https://img.shields.io/pypi/v/valx) +[![PyPI provenance](https://img.shields.io/badge/PyPI-view%20provenance-3775A9?logo=pypi&logoColor=white)](https://pypi.org/project/valx/#security) An open-source Python library for data cleaning tasks. It includes functions for profanity detection, and removal, and detection and removal of personal information. Also includes hate speech and offensive language detection and removal, using AI. -> [!IMPORTANT] -> If you are using `scikit-learn` versions older than version `1.3.0`, please also downgrade your version of `numpy` as stated below. Otherwise, you can continue to use your preferred version of `scikit-learn` without downgrading `numpy`. -> -> Please downgrade to `numpy` version `1.26.4`. Our ValX **DecisionTreeClassifier** AI model, relies on lower versions of `numpy`, because it was trained on these versions. -> For more information see: https://techoverflow.net/2024/07/23/how-to-fix-numpy-dtype-size-changed-may-indicate-binary-incompatibility-expected-96-from-c-header-got-88-from-pyobject/ +## Changes in 0.2.7 -> [!NOTE] -> ValX will automatically install a version of `scikit-learn` that is compatible with your device if you don't have one already. +- Sensitive information redaction now uses exact match positions on each line and merges overlaps, so it won't redact unrelated copies of a substring or leave part of an overlapping match exposed. Parenthesized phone area codes and signed coordinates are included in the matches. +- Profanity filtering supports the documented Finnish `FI` alias and punctuation in custom terms. Detections are in text order and the `Word` value preserves the original casing. Lines surrounded by `$` are no longer silently dropped. +- The bundled hate-speech models load on first use instead of at import time. Package metadata now reflects the Python 3.9 minimum already required by ValX's type annotations. +- The release workflow builds and tests the wheel and source distribution before publishing both to PyPI with signed attestations. ## Changes in 0.2.6 @@ -85,16 +84,7 @@ pip install valx ## Supported Python Versions -ValX supports the following Python versions: - -- Python 3.6 -- Python 3.7 -- Python 3.8 -- Python 3.9 -- Python 3.10 -- Python 3.11/Later (Preferred) - -Please ensure that you have one of these Python versions installed before using ValX. ValX may not work as expected on lower versions of Python than the supported. +ValX requires Python 3.9 or newer. CI covers Python 3.9 through 3.14. ## Features @@ -123,6 +113,7 @@ Below is a complete list of all the available supported languages for ValX's pro - Esperanto - EO - Persian +- FA - Finnish - FI - Filipino @@ -210,7 +201,7 @@ results_custom_plus_english = detect_profanity(sample_text, language="English", # Language will be "Custom + English" # [ # {'Line': 1, 'Column': 15, 'Word': 'custombadword1', 'Language': 'Custom + English'}, -# {'Line': 1, 'Column': 43, 'Word': 'asshole', 'Language': 'Custom + English'} +# {'Line': 1, 'Column': 39, 'Word': 'asshole', 'Language': 'Custom + English'} # ] print(results_custom_plus_english) ``` @@ -253,11 +244,11 @@ print(results_file_only) **Output Format for `detect_profanity`** -The `detect_profanity` function returns a list of dictionaries. Each dictionary includes: +The `detect_profanity` function returns non-overlapping matches in text order. When two terms match at the same position, the longer one wins. Each dictionary includes: - `"Line"`: The line number (1-indexed). - `"Column"`: The column number (1-indexed) where the profanity starts. -- `"Word"`: The detected profanity word. +- `"Word"`: The exact text that matched, with its original casing. - `"Language"`: Indicates the source of the word list: - `` (e.g., "English"): If only a built-in language list was used. - `"Custom"`: If `language=None` and only a `custom_words_list` was used. @@ -291,26 +282,23 @@ print(cleaned_text) The `load_profanity_words` function (used internally) also accepts `language` and `custom_words_list` if you need direct access to the word lists. -### Detect Sensitive Information +### Detect and Remove Sensitive Information ```python -from valx import detect_sensitive_information +from valx import detect_sensitive_information, remove_sensitive_information -# Detect sensitive information -detected_sensitive_info = detect_sensitive_information(sample_text) -``` +lines = ["Contact help@example.com or call (555) 123-4567."] +print(detect_sensitive_information(lines, info_type="email")) +# [(1, 8, 'Email', 'help@example.com')] -> [!NOTE] -> We have updated this function, and it now includes an optional argument for `info_type`, which can be used to detect only specific types of sensitive information. It was also added to `remove_sensitive_information`. +print(remove_sensitive_information(lines, info_type=["email", "phone"])) +# ['Contact [SENSITIVE] or call [SENSITIVE].'] +``` -### Remove Sensitive Information +Pass one type, a list of types, or omit `info_type` to scan for all types. Detection reports a one-based line number and a **zero-based** column. Removal merges overlapping matches before inserting `[SENSITIVE]`. -```python -from valx import remove_sensitive_information - -# Remove sensitive information -cleaned_text = remove_sensitive_information(sample_text2) -``` +> [!IMPORTANT] +> These are pattern-based filters, not a guarantee of anonymization. They can miss sensitive values or flag ordinary text. For example, `file_path` looks for filenames with an extension and can miss filenames containing spaces. Review redacted text before sharing it. ### Detect Hate Speech And Offensive Language @@ -331,6 +319,16 @@ outcome_of_detection = detect_hate_speech("You are stupid.") > [!NOTE] > See our [official documentation](https://infinitode-docs.gitbook.io/documentation/package-documentation/valx-package-documentation) for more examples on how to use **ValX**. +### Model compatibility + +Only the hate-speech functions load the bundled scikit-learn models. Scikit-learn can warn when loading a model saved with a different version, and compatibility across versions is not guaranteed. If you use scikit-learn 1.2.2, use NumPy 1.x with it. The rule-based profanity and sensitive-information filters do not load either model. + +## PyPI provenance + +Releases published by the Trusted Publishing workflow attach a [PEP 740 publish attestation](https://docs.pypi.org/attestations/publish/v1/) to each distribution file. The [PyPI Security tab](https://pypi.org/project/valx/#security) shows the status and lets you inspect the wheel and source archive separately. Older uploads are not retroactively attested. + +For setup and verification steps, see the [maintainer release guide](https://github.com/Infinitode/ValX/blob/main/CONTRIBUTING.md#releasing-to-pypi). + ## Contributing Contributions are welcome! If you encounter any issues, have suggestions, or want to contribute to ValX, please open an issue or submit a pull request on [GitHub](https://github.com/infinitode/valx). diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e51852a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,37 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "valx" +version = "0.2.7" +description = "Text filters for profanity, sensitive information, and offensive speech." +readme = "README.md" +requires-python = ">=3.9" +license = "LicenseRef-ValX-Modified-MIT" +license-files = ["LICENSE"] +authors = [{ name = "Infinitode Pty Ltd", email = "infinitode.ltd@gmail.com" }] +dependencies = ["packaging", "scikit-learn"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] + +[project.urls] +Repository = "https://github.com/Infinitode/ValX" +Issues = "https://github.com/Infinitode/ValX/issues" +Documentation = "https://infinitode-docs.gitbook.io/documentation/package-documentation/valx-package-documentation" + +[tool.setuptools] +packages = ["valx"] +include-package-data = false + +[tool.setuptools.package-data] +valx = ["models/*.pkl", "models/*.sav"] diff --git a/requirements.txt b/requirements.txt index ff88936..d6e1198 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -scikit-learn \ No newline at end of file +-e . diff --git a/setup.py b/setup.py deleted file mode 100644 index aebcf64..0000000 --- a/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -from setuptools import setup, find_packages - -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - -setup( - name='valx', - version='{{VERSION_PLACEHOLDER}}', - author='Infinitode Pty Ltd', - author_email='infinitode.ltd@gmail.com', - description='An open-source Python library for data cleaning tasks. Includes profanity detection, and removal. Now includes offensive language and hate speech detection using an AI model.', - long_description=long_description, - long_description_content_type='text/markdown', - url='https://github.com/infinitode/valx', - packages=find_packages(), - package_data={'valx': ['models/*']}, - install_requires=[ - 'scikit-learn' - ], - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - ], - python_requires='>=3.6', -) diff --git a/test.py b/test.py index d773b45..870a249 100644 --- a/test.py +++ b/test.py @@ -1,245 +1,172 @@ -from valx import ( - detect_profanity, - remove_profanity, - detect_sensitive_information, - remove_sensitive_information, - detect_hate_speech, - remove_hate_speech, - load_custom_profanity_from_file # Import the new function -) -import os # For file operations in tests - -# Define a helper to make assertions and print clear test results -def run_test(test_name, condition, success_message, failure_message): - print(f"--- Running Test: {test_name} ---") - if condition: - print(f"PASS: {success_message}") - else: - print(f"FAIL: {failure_message}") - assert condition, failure_message - print("-" * 30 + "\n") - -def main(): - print("Starting ValX tests...\n") - - # --- Original Profanity Tests --- - sample_text_orig = [ - "This is a sample text containing some profanity like porn, fuck, and asshole.", - "This line doesn't contain any profanity.", - "But this one has another, just in another language: oslošoust." - ] - detected_profanity_orig = detect_profanity(sample_text_orig, language='All') - run_test( - "Original Profanity Detection (All Languages)", - len(detected_profanity_orig) == 4, # porn, fuck, asshole, oslošoust - f"Detected {len(detected_profanity_orig)} profanities as expected.", - f"Expected 4 profanities, but found {len(detected_profanity_orig)}. Detected: {[d['Word'] for d in detected_profanity_orig]}" - ) - print("Detected (Original):", detected_profanity_orig) - print("Detected words (Original):", [d['Word'] for d in detected_profanity_orig]) - - cleaned_text_orig_file = "text_cleaned_original.txt" - remove_profanity(sample_text_orig, cleaned_text_orig_file, language="All") - run_test( - "Original Profanity Removal (All Languages)", - os.path.exists(cleaned_text_orig_file), - f"Cleaned file '{cleaned_text_orig_file}' created.", - f"Cleaned file '{cleaned_text_orig_file}' not found." - ) - if os.path.exists(cleaned_text_orig_file): - os.remove(cleaned_text_orig_file) # Clean up test file - - # --- Custom Profanity List Tests --- - custom_profanity_list = ["custombadword1", "supersecretcurse", "anotherone"] - custom_test_text = [ - "Here is custombadword1 and a normal word.", - "This uses supersecretcurse from the custom list.", - "Also, asshole from built-in English list.", - "And testwordalpha from the file list, plus anotherone here." - ] - - # 1. Custom list (Python list) + Specific Language (English) - # custom_profanity_list here is ["custombadword1", "supersecretcurse", "anotherone"] - # custom_test_text now contains "anotherone" - # So, "custombadword1", "supersecretcurse", "anotherone" (from custom list) and "asshole" (from English) should be detected. - detected_custom_english = detect_profanity(custom_test_text, language="English", custom_words_list=custom_profanity_list) - detected_words_custom_english = sorted([d['Word'] for d in detected_custom_english]) - expected_words_custom_english = sorted(["custombadword1", "supersecretcurse", "anotherone", "asshole"]) - run_test( - "Detect: Custom List + English", - detected_words_custom_english == expected_words_custom_english and \ - all(d['Language'] == "Custom + English" for d in detected_custom_english), - f"Correctly detected: {detected_words_custom_english} with language 'Custom + English'.", - f"Expected {expected_words_custom_english} with 'Custom + English', got {detected_words_custom_english}. Detected full: {detected_custom_english}" - ) - print("Detected (Custom List + English):", detected_custom_english) - - # 2. Custom list (Python list) + language=None - # custom_profanity_list here is ["custombadword1", "supersecretcurse", "anotherone"] - # "asshole" should not be detected here as we are not using the English list - # "testwordalpha" is in the text, but not in this custom_profanity_list. - detected_custom_only = detect_profanity(custom_test_text, language=None, custom_words_list=custom_profanity_list) - detected_words_custom_only = sorted([d['Word'] for d in detected_custom_only]) - expected_words_custom_only = sorted(["custombadword1", "supersecretcurse", "anotherone"]) # "anotherone" is in this custom list - run_test( - "Detect: Custom List Only (language=None)", - detected_words_custom_only == expected_words_custom_only and \ - all(d['Language'] == "Custom" for d in detected_custom_only), - f"Correctly detected: {detected_words_custom_only} with language 'Custom'.", - f"Expected {expected_words_custom_only} with 'Custom', got {detected_words_custom_only}. Detected full: {detected_custom_only}" - ) - print("Detected (Custom List Only):", detected_custom_only) - - # --- Custom Profanity File Tests --- - custom_file_path = "custom_profanity.txt" # Created in previous step - - # Ensure the test file exists - if not os.path.exists(custom_file_path): - print(f"CRITICAL FAIL: Test file '{custom_file_path}' not found. Skipping file-based tests.") - else: - custom_words_from_file = load_custom_profanity_from_file(custom_file_path) - expected_file_words = ["custombadword1", "supersecretcurse", "anotherone", "testwordalpha", "testwordbeta"] - run_test( - "Load Custom Profanity from File", - sorted(custom_words_from_file) == sorted(expected_file_words), - f"Successfully loaded {len(custom_words_from_file)} words from file.", - f"Expected {expected_file_words}, got {sorted(custom_words_from_file)}" +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +import valx + + +class ProfanityTests(unittest.TestCase): + def test_language_names_and_codes_are_case_insensitive(self): + for language, word in ( + ("enGliSh", "porn"), ("eN", "porn"), ("ES", "gilipollas"), + ("Finnish", "perkele"), ("fI", "perkele"), + ): + with self.subTest(language=language): + self.assertEqual( + valx.detect_profanity([word], language), + [{"Line": 1, "Column": 1, "Word": word, "Language": language}], + ) + + def test_all_languages(self): + lines = ["Some words: porn, asshole.", "Another language: oslošoust."] + matches = valx.detect_profanity(lines, "All") + self.assertEqual([match["Word"] for match in matches], ["porn", "asshole", "oslošoust"]) + self.assertEqual( + valx.remove_profanity(lines, language="All"), + ["Some words: bad word, bad word.", "Another language: bad word."], ) - print("Words loaded from file:", custom_words_from_file) - - # 3. Custom list (from file) + Specific Language (English) - # Text includes "asshole" (English) and "testwordalpha" (file) - detected_file_english = detect_profanity(custom_test_text, language="English", custom_words_list=custom_words_from_file) - detected_words_file_english = sorted([d['Word'] for d in detected_file_english]) - # custombadword1, supersecretcurse, asshole, testwordalpha, anotherone (since "anotherone" is in custom_test_text and custom_words_from_file) - expected_words_file_english = sorted(["custombadword1", "supersecretcurse", "asshole", "testwordalpha", "anotherone"]) - run_test( - "Detect: Custom File List + English", - detected_words_file_english == expected_words_file_english and \ - all(d['Language'] == "Custom + English" for d in detected_file_english), - f"Correctly detected: {detected_words_file_english} with 'Custom + English'.", - f"Expected {expected_words_file_english} with 'Custom + English', got {detected_words_file_english}. Detected full: {detected_file_english}" + + def test_custom_lists_alone_or_combined(self): + lines = ["Before BLOCKED and porn.", "blocked again, unblocked is fine."] + words = [" blocked ", "blocked"] + for language, label, expected in ( + (None, "Custom", [(1, 8, "BLOCKED"), (2, 1, "blocked")]), + ("EN", "Custom + EN", [(1, 8, "BLOCKED"), (1, 20, "porn"), (2, 1, "blocked")]), + ("All", "Custom + All", [(1, 8, "BLOCKED"), (1, 20, "porn"), (2, 1, "blocked")]), + ): + with self.subTest(language=language): + self.assertEqual( + valx.detect_profanity(lines, language, words), + [dict(Line=line, Column=col, Word=word, Language=label) + for line, col, word in expected], + ) + + def test_punctuation_and_longest_match(self): + lines = ["dummy phrase, #TAG, C++ and C++17."] + words = ["dummy", "dummy phrase", "#tag", "C++", "word"] + matches = valx.detect_profanity(lines, None, words) + self.assertEqual([match["Word"] for match in matches], ["dummy phrase", "#TAG", "C++"]) + self.assertEqual( + valx.remove_profanity(lines, language=None, custom_words_list=words), + ["bad word, bad word, bad word and C++17."], ) - print("Detected (Custom File + English):", detected_file_english) - - # 4. Custom list (from file) + language=None - # Text includes "testwordalpha" (file) but "asshole" (English) should be ignored - # custom_words_from_file includes "custombadword1", "supersecretcurse", "anotherone", "testwordalpha", "testwordbeta" - # custom_test_text includes "custombadword1", "supersecretcurse", "anotherone", "testwordalpha" - detected_file_only = detect_profanity(custom_test_text, language=None, custom_words_list=custom_words_from_file) - detected_words_file_only = sorted([d['Word'] for d in detected_file_only]) - expected_words_file_only = sorted(["custombadword1", "supersecretcurse", "testwordalpha", "anotherone"]) - run_test( - "Detect: Custom File List Only (language=None)", - detected_words_file_only == expected_words_file_only and \ - all(d['Language'] == "Custom" for d in detected_file_only), - f"Correctly detected: {detected_words_file_only} with language 'Custom'.", - f"Expected {expected_words_file_only} with 'Custom', got {detected_words_file_only}. Detected full: {detected_file_only}" + + def test_dollar_delimited_input_is_not_dropped(self): + lines = ["$notes$", "$blocked$", ""] + words = ["blocked"] + self.assertEqual( + valx.detect_profanity(lines, None, words), + [{"Line": 2, "Column": 2, "Word": "blocked", "Language": "Custom"}], + ) + self.assertEqual( + valx.remove_profanity(lines, language=None, custom_words_list=words), + ["$notes$", "$bad word$", ""], ) - print("Detected (Custom File Only):", detected_file_only) - - # 5. Custom list (from file) + language='All' - # This should detect "oslošoust" (from 'All') and words from file that are in combined_text_for_all - combined_text_for_all = sample_text_orig + custom_test_text - detected_file_all = detect_profanity(combined_text_for_all, language="All", custom_words_list=custom_words_from_file) - detected_words_file_all = sorted(list(set([d['Word'] for d in detected_file_all]))) # Use set for uniqueness - expected_words_file_all = sorted(list(set(["porn", "fuck", "asshole", "oslošoust", "custombadword1", "supersecretcurse", "anotherone", "testwordalpha"]))) - run_test( - "Detect: Custom File List + All Languages", - detected_words_file_all == expected_words_file_all and \ - all(d['Language'] == "Custom + All" for d in detected_file_all), - f"Correctly detected words with 'Custom + All'. Words: {detected_words_file_all}", - f"Expected words with 'Custom + All'. Expected: {expected_words_file_all}, Got: {detected_words_file_all}. Detected full: {detected_file_all}" + + def test_custom_wordlist_validation(self): + for words in (None, [], ["", " "]): + with self.subTest(words=words), self.assertRaises(ValueError): + valx.load_profanity_words(None, words) + self.assertEqual(valx.detect_profanity(["hello"], custom_words_list=[""]), []) + with self.assertRaises(TypeError): + valx.load_profanity_words(None, "blocked") + + def test_returned_wordlist_can_be_modified_without_affecting_other_calls(self): + words = valx.load_profanity_words("FI") + self.assertIn("perkele", words) + words.clear() + self.assertIn("perkele", valx.load_profanity_words("Finnish")) + + def test_load_custom_file_and_write_cleaned_text(self): + with tempfile.TemporaryDirectory() as temp: + source = Path(temp) / "custom.txt" + source.write_text("# comment\n\n café\n # other comment\n two words\n", encoding="utf-8-sig") + words = valx.load_custom_profanity_from_file(source) + self.assertEqual(words, ["café", "two words"]) + output = Path(temp) / "cleaned.txt" + self.assertEqual(valx.remove_profanity(["CAFÉ!"], output, None, words), ["bad word!"]) + self.assertEqual(output.read_text(encoding="utf-8"), "bad word!") + with self.assertRaises(FileNotFoundError): + valx.load_custom_profanity_from_file(Path(temp) / "missing.txt") + + +class SensitiveInformationTests(unittest.TestCase): + def test_detect_specific_type_and_preserve_zero_based_column(self): + self.assertEqual( + valx.detect_sensitive_information(["No contact", "Contact x@example.com or (555) 123-4567."], "email"), + [(2, 8, "Email", "x@example.com")], ) - print("Detected (Custom File + All):", detected_file_all) - - # Test remove_profanity with custom list from file - cleaned_output_file_custom = "text_cleaned_custom_file.txt" - text_to_clean_custom = [ - "This has custombadword1 and testwordalpha.", - "Also regular profanity like asshole." - ] - remove_profanity(text_to_clean_custom, cleaned_output_file_custom, language="English", custom_words_list=custom_words_from_file) - run_test( - "Remove: Custom File List + English", - os.path.exists(cleaned_output_file_custom), - f"Cleaned file '{cleaned_output_file_custom}' created using custom file list + English.", - f"Cleaned file '{cleaned_output_file_custom}' not found." + + def test_phone_with_parentheses_is_fully_redacted(self): + lines = ["Call (555) 123-4567.", "Call(555) 123-4567."] + self.assertEqual( + valx.detect_sensitive_information(lines, "phone"), + [(1, 5, "Phone", "(555) 123-4567"), (2, 4, "Phone", "(555) 123-4567")], ) - if os.path.exists(cleaned_output_file_custom): - with open(cleaned_output_file_custom, 'r') as f: - content = f.read() - # Check if custom and built-in words are replaced - cond = "custombadword1" not in content and \ - "testwordalpha" not in content and \ - "asshole" not in content and \ - "bad word" in content # Check if replacement marker is present - run_test( - "Remove: Content Check (Custom File + English)", - cond, - "Profanity correctly replaced in output file.", - f"Profanity not correctly replaced. Content:\n{content}" + self.assertEqual( + valx.remove_sensitive_information(lines, info_type="phone"), + ["Call [SENSITIVE].", "Call[SENSITIVE]."], + ) + + def test_negative_coordinates_include_the_sign(self): + lines = ["Site: -25.7, +28.2."] + self.assertEqual(valx.detect_sensitive_information(lines, "geo_coords"), [(1, 6, "Geo_coords", "-25.7, +28.2")]) + self.assertEqual(valx.remove_sensitive_information(lines, info_type="geo_coords"), ["Site: [SENSITIVE]."]) + self.assertEqual(valx.detect_sensitive_information(["91.1, 18", "25, 180.1"], "geo_coords"), []) + + def test_redacts_only_matched_spans_even_with_an_iterator(self): + lines = ["ID 123-45-6789, not X123-45-6789Y.", "X123-45-6789Y", "123-45-6789"] + expected = ["ID [SENSITIVE], not X123-45-6789Y.", "X123-45-6789Y", "[SENSITIVE]"] + for text_data in (lines, iter(lines)): + self.assertEqual(valx.remove_sensitive_information(text_data, info_type="ssn"), expected) + + def test_overlapping_matches_are_redacted_once_in_either_type_order(self): + for info_type in (["username", "email"], ["email", "username"]): + with self.subTest(info_type=info_type): + self.assertEqual( + valx.remove_sensitive_information(["Mail me@example.com or ping @help."], info_type=info_type), + ["Mail [SENSITIVE] or ping [SENSITIVE]."], ) - os.remove(cleaned_output_file_custom) # Clean up - - # --- Test language=None and custom_words_list=None for load_profanity_words (should raise ValueError) --- - try: - detect_profanity(["some text"], language=None, custom_words_list=None) - run_test("Error Handling: language=None, custom_words_list=None", False, "", "ValueError was not raised when language and custom_words_list are both None.") - except ValueError: - run_test("Error Handling: language=None, custom_words_list=None", True, "ValueError correctly raised.", "") - except Exception as e: - run_test("Error Handling: language=None, custom_words_list=None", False, "", f"Unexpected error raised: {e}") - - - # --- Original Sensitive Info and AI Hate Speech Tests (copied from original test.py) --- - print("\n--- Running Original Sensitive Info and AI Hate Speech Tests ---") - ai_sample_text = [ - "This is a sample text containing some profanity like porn, fuck, and asshole.", # Will be removed by AI - "This line doesn't contain any profanity.", # Kept - "But this one has another, just in another language: oslošoust." # Will be removed by AI - ] - print("AI detect_hate_speech('You're so stupid.'):", detect_hate_speech("You're so stupid.")) # Expected: Offensive Speech - - sample_text2 = [ - "Please contact john.doe@example.com or call 555-123-4567 for more information.", - "We will need your credit card number to complete the transaction: 1234-5678-9012-3456.", - "My social security number is 123-45-6789 and my ID number is AB123456.", - "Our office address is 123 Main St, Anytown, USA. Please visit us!", - "Your IP address is 192.168.1.1. Please don't share it with anyone." - ] - - detected_sensitive_info = detect_sensitive_information(sample_text2) - print("Detected sensitive information:") - for line_num, col_num, info_type, info in detected_sensitive_info: - print(f"Line {line_num}, Column {col_num}: {info_type} - {info}") - - cleaned_text_sensitive = remove_sensitive_information(sample_text2) - print("\nCleaned text (sensitive info removed):") - for line in cleaned_text_sensitive: - print(line) - - cleaned_text_ai = remove_hate_speech(ai_sample_text) # Use ai_sample_text - print("\nCleaned text using AI (hate/offensive speech removed):") # Expected: only the clean line - for line in cleaned_text_ai: - print(line) - - # Based on current observation, the AI model in this env/version setup - # flags the first two lines of ai_sample_text and not the third. - expected_ai_cleaned_lines = [ - "But this one has another, just in another language: oslošoust." - ] - condition_ai_removal = len(cleaned_text_ai) == len(expected_ai_cleaned_lines) and \ - all(expected_line in cleaned_text_ai for expected_line in expected_ai_cleaned_lines) - - run_test( - "AI Hate Speech Removal", - condition_ai_removal, - "AI correctly processed lines based on its observed hate/offensive speech detection in this environment.", - f"AI removal incorrect. Expected {len(expected_ai_cleaned_lines)} specific lines, got {len(cleaned_text_ai)}. Expected: {expected_ai_cleaned_lines}, Got: {cleaned_text_ai}" - ) - - print("\nAll ValX tests completed.") + + def test_partly_overlapping_matches_redact_their_union(self): + self.assertEqual( + valx.remove_sensitive_information(["1234 5678 9012 345 678 9012"], info_type=["credit_card", "phone"]), + ["[SENSITIVE]"], + ) + + def test_default_patterns_do_not_treat_email_parts_as_file_paths(self): + lines = ["Mail first.last@example.com, then call (555) 123-4567."] + self.assertEqual(valx.detect_sensitive_information(lines, "file_path"), []) + self.assertEqual(valx.remove_sensitive_information(lines), ["Mail [SENSITIVE], then call [SENSITIVE]."]) + + def test_file_paths_and_output_file(self): + lines = [r"Open /var/log/app.log and C:\Program Files\Reports\report.txt or report.csv."] + expected = ["Open [SENSITIVE] and [SENSITIVE] or [SENSITIVE]."] + with tempfile.TemporaryDirectory() as temp: + output = Path(temp) / "redacted.txt" + self.assertEqual(valx.remove_sensitive_information(lines, output, "file_path"), expected) + self.assertEqual(output.read_text(encoding="utf-8"), expected[0]) + + +class HateSpeechTests(unittest.TestCase): + def test_text_filters_do_not_import_sklearn(self): + subprocess.run( + [sys.executable, "-c", "import sys, valx; valx.detect_profanity(['hello']); " + "assert 'sklearn' not in sys.modules"], + cwd=Path(valx.__file__).resolve().parent.parent, + check=True, + ) + + def test_bundled_model_prediction_and_filtering(self): + offensive = "You're so stupid." + clean = "Good morning, have a nice day." + self.assertEqual(valx.detect_hate_speech(offensive).tolist(), ["Offensive Speech"]) + self.assertEqual(valx.detect_hate_speech(clean).tolist(), ["No Hate and Offensive Speech"]) + self.assertEqual(valx.remove_hate_speech([offensive, clean]), [clean]) + self.assertEqual(valx.remove_hate_speech([]), []) + if __name__ == "__main__": - main() + unittest.main() diff --git a/valx/__init__.py b/valx/__init__.py index fc42d70..1757c3e 100644 --- a/valx/__init__.py +++ b/valx/__init__.py @@ -1,37 +1,22 @@ -import os -import re - -# New version, that includes AI detection -import pickle -from sklearn import __version__ as sklearnversion -from sklearn.tree import DecisionTreeClassifier # Import the DecisionTreeClassifier class -from sklearn.feature_extraction.text import CountVectorizer # Import the CountVectorizer class +"""Text filters for profanity, sensitive information, and offensive speech. -''' ValX uses data from this GitHub repository: https://github.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/ © 2012-2020 Shutterstock, Inc. Creative Commons Attribution 4.0 International License: https://github.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/blob/master/LICENSE -''' +""" -def load_profanity_words(language='English', custom_words_list: list[str] = None): - """ - Load profanity words for the specified language, optionally including a custom list. - - Args: - language (str, optional): The language for which to load profanity words. - Defaults to 'English'. Can be None to use only custom_words_list. - custom_words_list (list[str], optional): A list of custom profanity words. Defaults to None. +import os +import pickle +import re +from functools import lru_cache +from typing import Optional - Returns: - list: A list of profanity words. - Raises: - ValueError: If language is None and custom_words_list is None or empty. - """ - # Profanity words as a string +@lru_cache(maxsize=1) +def _builtin_profanity_lists(): PROFANITY_WORDS = """ $Arabic|AR$ سكس @@ -810,7 +795,7 @@ def load_profanity_words(language='English', custom_words_list: list[str] = None لش منی هرزه - $Finnish$ + $Finnish|FI$ alfred nussi bylsiä haahka @@ -2729,302 +2714,225 @@ def load_profanity_words(language='English', custom_words_list: list[str] = None 硬膠 """ - profanity_lists = {} - current_language = None + profanity_lists = {"all": []} current_words = [] - - for line in PROFANITY_WORDS.split('\n'): + for line in PROFANITY_WORDS.splitlines(): line = line.strip() if line.startswith("$") and line.endswith("$"): - if current_language: - for lang in current_language.split('|'): - profanity_lists[lang.strip().lower()] = current_words - current_words = [] - current_language = line.strip("$") + current_words = [] + for alias in line[1:-1].split('|'): + profanity_lists[alias.strip().lower()] = current_words elif line: current_words.append(line) + profanity_lists['all'].append(line) - if current_language: - for lang in current_language.split('|'): - profanity_lists[lang.strip().lower()] = current_words - - loaded_words = [] - if language is None: - if not custom_words_list: - raise ValueError("If language is None, custom_words_list must be provided and non-empty.") - loaded_words = list(custom_words_list) # Use a copy - elif language.lower() == "all": - for words in profanity_lists.values(): - loaded_words.extend(words) - if custom_words_list: - loaded_words.extend(w for w in custom_words_list if w not in loaded_words) - else: - loaded_words = list(profanity_lists.get(language.lower(), [])) # Use a copy - if custom_words_list: - loaded_words.extend(w for w in custom_words_list if w not in loaded_words) + return {alias: tuple(dict.fromkeys(words)) for alias, words in profanity_lists.items()} - return list(set(loaded_words)) # Return unique words +def _profanity_words(language, custom_words_list): + custom_words = [] + if custom_words_list is not None: + if isinstance(custom_words_list, str): + raise TypeError("custom_words_list must be a list of strings, not a single string.") + for word in custom_words_list: + if not isinstance(word, str): + raise TypeError("Each custom profanity word must be a string.") + word = word.strip() + if word: + custom_words.append(word) -def load_custom_profanity_from_file(filepath: str) -> list[str]: - """ - Loads a custom list of profanity words from a file. + if language is None and not custom_words: + raise ValueError("If language is None, provide at least one non-blank custom profanity word.") - The file should contain one word per line. Lines starting with '#' will be ignored as comments. - Empty lines or lines with only whitespace will also be ignored. + built_in = () if language is None else _builtin_profanity_lists().get(language.strip().lower(), ()) + return tuple(dict.fromkeys((*built_in, *custom_words))), bool(custom_words) - Args: - filepath (str): The path to the file containing custom profanity words. - Returns: - list[str]: A list of words loaded from the file. +def load_profanity_words(language='English', custom_words_list: Optional[list[str]] = None) -> list[str]: + """Return unique words for a language (or 'All'), optionally with custom words. - Raises: - FileNotFoundError: If the specified filepath does not exist. - IOError: For other file reading issues. + Custom words are stripped of surrounding whitespace; blank entries are ignored. + If language is None, at least one non-blank custom word is required. """ - custom_words = [] - try: - with open(filepath, 'r', encoding='utf-8') as f: - for line in f: - word = line.strip() - if word and not word.startswith('#'): - custom_words.append(word) - except FileNotFoundError: - # Re-raise FileNotFoundError to be explicit, or handle as per specific project policy - # For example, could log a warning and return empty list: - # print(f"Warning: File not found at {filepath}. Returning empty custom word list.") - # return [] - raise - except IOError as e: - # Handle other IO errors, perhaps log them or raise a more generic error - # print(f"Warning: Error reading file {filepath}: {e}. Returning empty custom word list.") - # return [] - raise IOError(f"Error reading custom profanity file {filepath}: {e}") - return custom_words + words, _ = _profanity_words(language, custom_words_list) + return list(words) -def detect_profanity(text_data, language='English', custom_words_list: list[str] = None): - """ - Detect profanity in text data using regex. +def load_custom_profanity_from_file(filepath: str) -> list[str]: + """Load a UTF-8 file of custom words, one entry per line. - Args: - text_data (list): A list of strings representing the text data to analyze. - language (str, optional): The language used to detect profanity. Defaults to 'English'. - Can be None if custom_words_list is provided. - Available languages include: All, Arabic, AR, Czech, CS, Danish, DA, German, DE, - English, EN, Esperanto, EO, Persian, Finnish, FI, Filipino, FIL, French, FR, - French (CA), FR-CA-U-SD-CAQC, Hindi, HI, Hungarian, HU, Italian, IT, Japanese, JA, Kabyle, KAB, - Korean, KO, Dutch, NL, Norwegian, NO, Polish, PL, Portuguese, PT, Russian, RU, Spanish, ES, - Swedish, SV, Thai, TH, Klingon, TLH, Turkish, TR, Chinese, ZH. - custom_words_list (list[str], optional): A list of custom profanity words. Defaults to None. - Returns: - list: A list of dictionaries where each dictionary represents a detected instance of profanity. - Each dictionary contains the following keys: - - "Line" (int): The line number where the profanity was detected. - - "Column" (int): The column number (position in the line) where the profanity starts. - - "Word" (str): The detected profanity word. - - "Language" (str): The language string indicating the source of the profanity list. + Blank lines and lines starting with '#' after stripping whitespace are + ignored. File reading errors propagate to the caller. """ - profanity_keywords = load_profanity_words(language, custom_words_list) - detected = set() # Use a set to store unique occurrences - - display_language = "Custom" - if language: - if custom_words_list: - display_language = f"Custom + {language}" - else: - display_language = language - elif not custom_words_list: - # This case should ideally be prevented by load_profanity_words raising an error, - # but as a fallback, if somehow load_profanity_words returns empty and language is None - # and no custom_words_list, we avoid error here. - return [] - - - for i, line in enumerate(text_data): - # Skip lines with language markers (though less relevant if custom lists are primary) - if line.startswith("$") and line.endswith("$"): - continue + custom_words = [] + with open(filepath, 'r', encoding='utf-8-sig') as f: + for line in f: + word = line.strip() + if word and not word.startswith('#'): + custom_words.append(word) + return custom_words - # Detect profanity - for profanity in profanity_keywords: - matches = re.finditer(r'\b{}\b'.format(re.escape(profanity)), line, flags=re.IGNORECASE) - for match in matches: - # Store with display_language, not the input `language` parameter - detected.add((i + 1, match.start() + 1, display_language, profanity)) - detections = [] # Initialize detections as a list +@lru_cache(maxsize=32) +def _profanity_pattern(words): + if not words: + return None + # Escape literal terms and prefer a phrase over a shorter word at the same position. + alternatives = '|'.join(re.escape(word) for word in sorted(words, key=len, reverse=True)) + return re.compile(rf'(?= (1, 3, 0): - DECISION_TREE_MODEL_PATH = os.path.join(MODEL_DIR, UPGRADED_MODEL_FILENAME) -else: - DECISION_TREE_MODEL_PATH = os.path.join(MODEL_DIR, DEFAULT_MODEL_FILENAME) +@lru_cache(maxsize=1) +def _load_hate_speech_models(): + from packaging.version import Version + from sklearn import __version__ + + # scikit-learn 1.3 changed the serialized decision-tree layout. + filename = 'classifier_upgraded.pkl' if Version(__version__).release >= (1, 3) else 'decision_tree_model.sav' + with open(os.path.join(MODEL_DIR, filename), 'rb') as model_file: + model = pickle.load(model_file) + with open(os.path.join(MODEL_DIR, 'count_vectorizer.sav'), 'rb') as vectorizer_file: + vectorizer = pickle.load(vectorizer_file) + return model, vectorizer -# Load the saved models -model = pickle.load(open(DECISION_TREE_MODEL_PATH, 'rb')) -cv = pickle.load(open(COUNT_VECTORIZER_PATH, 'rb')) def detect_hate_speech(text): - """ - Detect offensive language or hate speech in the provided text string, using an AI model. + """Classify one string. Returns a one-element NumPy array of labels.""" + model, vectorizer = _load_hate_speech_models() + return model.predict(vectorizer.transform([text])) - Args: - text_data (list of str): A list of strings representing the text data to be cleaned. - - Returns: - list of str: A list of strings representing the outcome of the detection. - """ - text = cv.transform([text]).toarray() - return model.predict((text)) def remove_hate_speech(text_data): - """ - Remove offensive language or hate speech in the provided text data array, using an AI model. - - Args: - text (str): A string representing the text data to be used for hate speech detection and offensive language detection. - - Returns: - list of str: A list of strings representing the cleaned text data. - """ - cleaned_data = [] - for sentence in text_data: - outcome = detect_hate_speech(sentence) - if outcome != ['Hate Speech'] and outcome != ['Offensive Speech'] and outcome == ['No Hate and Offensive Speech']: - cleaned_data.append(sentence) - - return cleaned_data \ No newline at end of file + """Return the lines classified as neither hate speech nor offensive speech.""" + return [line for line in text_data if detect_hate_speech(line)[0] == 'No Hate and Offensive Speech'] From cafe3811b5cde5357a8617d508dc23256ad4effb Mon Sep 17 00:00:00 2001 From: johanbeytell1 <107624055+johanbeytell1@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:47:49 +0000 Subject: [PATCH 2/2] Drop Python 3.9 and clarify profanity language codes Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/test.yml | 2 +- CONTRIBUTING.md | 2 +- README.md | 97 ++++++++++++++------------------------ pyproject.toml | 3 +- test.py | 1 + 5 files changed, 39 insertions(+), 66 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f82b7a9..e668aae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bf0e596..fb0d35f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,7 +8,7 @@ Thank you for considering contributing to **ValX**! Contributions, whether they Create a fork of this repository to your GitHub account and clone it locally. 2. **Set Up the Environment** - Use Python 3.9 or newer. In a virtual environment, install ValX for development and run its tests: + Use Python 3.10 or newer. In a virtual environment, install ValX for development and run its tests: ```bash python -m pip install -r requirements.txt diff --git a/README.md b/README.md index ff71a94..7e16b59 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ValX -![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue.svg) +![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg) [![Code Size](https://img.shields.io/github/languages/code-size/infinitode/valx)](https://github.com/infinitode/valx) ![Downloads](https://pepy.tech/badge/valx) ![License Compliance](https://img.shields.io/badge/license-compliance-brightgreen.svg) @@ -13,7 +13,7 @@ An open-source Python library for data cleaning tasks. It includes functions for - Sensitive information redaction now uses exact match positions on each line and merges overlaps, so it won't redact unrelated copies of a substring or leave part of an overlapping match exposed. Parenthesized phone area codes and signed coordinates are included in the matches. - Profanity filtering supports the documented Finnish `FI` alias and punctuation in custom terms. Detections are in text order and the `Word` value preserves the original casing. Lines surrounded by `$` are no longer silently dropped. -- The bundled hate-speech models load on first use instead of at import time. Package metadata now reflects the Python 3.9 minimum already required by ValX's type annotations. +- The bundled hate-speech models load on first use instead of at import time. This release requires Python 3.10 or newer. - The release workflow builds and tests the wheel and source distribution before publishing both to PyPI with signed attestations. ## Changes in 0.2.6 @@ -84,7 +84,7 @@ pip install valx ## Supported Python Versions -ValX requires Python 3.9 or newer. CI covers Python 3.9 through 3.14. +ValX requires Python 3.10 or newer. CI covers Python 3.10 through 3.14. ## Features @@ -97,65 +97,38 @@ ValX requires Python 3.9 or newer. CI covers Python 3.9 through 3.14. ### List of supported languages for profanity detection and removal -Below is a complete list of all the available supported languages for ValX's profanity detection and removal functions which are valid values for `language`: - -- All -- Arabic -- AR -- Czech -- CS -- Danish -- DA -- German -- DE -- English -- EN -- Esperanto -- EO -- Persian -- FA -- Finnish -- FI -- Filipino -- FIL -- French -- FR -- French (CA) -- FR-CA-U-SD-CAQC -- Hindi -- HI -- Hungarian -- HU -- Italian -- IT -- Japanese -- JA -- Kabyle -- KAB -- Korean -- KO -- Dutch -- NL -- Norwegian -- NO -- Polish -- PL -- Portuguese -- PT -- Russian -- RU -- Spanish -- ES -- Swedish -- SV -- Thai -- TH -- Klingon -- TLH -- Turkish -- TR -- Chinese -- ZH +Pass a language name or code from the table as `language`. Both are case-insensitive. Use `All` to combine all built-in lists. + +| Language name | Code | +| --- | --- | +| Arabic | `AR` | +| Czech | `CS` | +| Danish | `DA` | +| German | `DE` | +| English | `EN` | +| Esperanto | `EO` | +| Persian | `FA` | +| Finnish | `FI` | +| Filipino | `FIL` | +| French | `FR` | +| French (CA) | `FR-CA-U-SD-CAQC` | +| Hindi | `HI` | +| Hungarian | `HU` | +| Italian | `IT` | +| Japanese | `JA` | +| Kabyle | `KAB` | +| Korean | `KO` | +| Dutch | `NL` | +| Norwegian | `NO` | +| Polish | `PL` | +| Portuguese | `PT` | +| Russian | `RU` | +| Spanish | `ES` | +| Swedish | `SV` | +| Thai | `TH` | +| Klingon | `TLH` | +| Turkish | `TR` | +| Chinese | `ZH` | ## Usage diff --git a/pyproject.toml b/pyproject.toml index e51852a..b2e5d20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "valx" version = "0.2.7" description = "Text filters for profanity, sensitive information, and offensive speech." readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" license = "LicenseRef-ValX-Modified-MIT" license-files = ["LICENSE"] authors = [{ name = "Infinitode Pty Ltd", email = "infinitode.ltd@gmail.com" }] @@ -16,7 +16,6 @@ classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", diff --git a/test.py b/test.py index 870a249..deb64c9 100644 --- a/test.py +++ b/test.py @@ -11,6 +11,7 @@ class ProfanityTests(unittest.TestCase): def test_language_names_and_codes_are_case_insensitive(self): for language, word in ( ("enGliSh", "porn"), ("eN", "porn"), ("ES", "gilipollas"), + ("Persian", "پورن"), ("fA", "پورن"), ("Finnish", "perkele"), ("fI", "perkele"), ): with self.subTest(language=language):