pyproject.toml

[project]
name = "openvino-tokenizers"
version = "2026.2.0.0"
description = "Convert tokenizers into OpenVINO models"
requires-python = ">=3.10"
readme = { file = "README.md", content-type="text/markdown" }
license = { "text" = "Apache-2.0" }
license-files = [
    "LICENSE",
    "third-party-programs.txt",
    "SECURITY.md",
    "README.md"
]
authors = [
    { name = "OpenVINO Developers", email = "openvino@intel.com" },
]

classifiers = [
    "Development Status :: 5 - Production/Stable",
    "License :: OSI Approved :: Apache Software License",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Software Development :: Libraries :: Python Modules",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Operating System :: Unix",
    "Operating System :: POSIX :: Linux",
    "Operating System :: Microsoft :: Windows",
    "Operating System :: MacOS",
    "Programming Language :: C++",
    "Programming Language :: C",
    "Programming Language :: Python :: 3 :: Only",
]

dependencies = [
    # support of nightly openvino packages with dev suffix
    "openvino~=2026.2.0.dev"
]

[project.optional-dependencies]
transformers = [
    "transformers[sentencepiece]>=4.36.0",
    "tiktoken>=0.3.0",
]

[project.scripts]
openvino_tokenizers = "openvino_tokenizers.cli_tools.main:main"
convert_tokenizer = "openvino_tokenizers.cli_tools.convert_tokenizer:convert_hf_tokenizer"

[tool.ruff]
line-length = 119

[tool.ruff.lint]
ignore = ["C901", "E501", "E741", "W605"]
select = ["C", "E", "F", "I", "W", "UP006"]

[tool.ruff.lint.per-file-ignores]
"__init__.py" = ["F401"]
"openvino_tokenizers/hf_parser.py" = ["F821"]

[tool.ruff.lint.isort]
lines-after-imports = 2

[tool.bandit]
tests = ["B301", "B302", "B303", "B304", "B305", "B306", "B308", "B310", "B311", "B312", "B313", "B314", "B315", "B316", "B317", "B318", "B319", "B321", "B323", "B324", "B401", "B402", "B403", "B404", "B405", "B406", "B407", "B408", "B409", "B411", "B412", "B413"]
skips = ["B101", "B102", "B103", "B104", "B105", "B106", "B107", "B108", "B110", "B112", "B201", "B501", "B502", "B503", "B504", "B505", "B506", "B507", "B601", "B602", "B603", "B604", "B605", "B606", "B607", "B608", "B609", "B610", "B611", "B701", "B702", "B703"]
no_shell = ["os.execl", "os.execle", "os.execlp", "os.execlpe", "os.execv", "os.execve", "os.execvp", "os.execvpe", "os.spawnl", "os.spawnle", "os.spawnlp", "os.spawnlpe", "os.spawnv", "os.spawnve", "os.spawnvp", "os.spawnvpe", "os.startfile"]
shell = ["os.system", "os.popen", "os.popen2", "os.popen3", "os.popen4", "popen2.popen2", "popen2.popen3", "popen2.popen4", "popen2.Popen3", "popen2.Popen4", "commands.getoutput", "commands.getstatusoutput"]
subprocess = ["subprocess.Popen", "subprocess.call", "subprocess.check_call", "subprocess.check_output", "subprocess.run"]
exclude_dirs = ["tests"]

[tool.py-build-cmake.module]
directory = "python"

[tool.py-build-cmake.sdist]
include = ["cmake", "python", "src", "CMakeLists.txt", "LICENSE", "README.md", "SECURITY.md", "third-party-programs.txt"]

[tool.py-build-cmake.cmake]
minimum_version = "3.15"
build_type = "Release"
config = ["Release"]
find_python3 = true
build_args = ["--parallel", "--target", "openvino_tokenizers"]
install_args = ["--strip"]
install_components = ["openvino_tokenizers", "openvino_tokenizers_python", "openvino_tokenizers_licenses"]

[tool.py-build-cmake.cmake.options]
OPENVINO_TOKENIZERS_INSTALL_BINDIR = "openvino_tokenizers/lib"
OPENVINO_TOKENIZERS_INSTALL_LIBDIR = "openvino_tokenizers/lib"

[tool.py-build-cmake.wheel]
python_tag = ['py3']
python_abi = "none"

[build-system]
requires = [
    "py-build-cmake==0.4.3",
    "cmake~=3.14",
    "openvino~=2026.2.0.dev"
]
build-backend = "py_build_cmake.build"

[tool.poetry]
package-mode = false

[[tool.poetry.source]]
name = "openvino-nightly"
url = "https://storage.openvinotoolkit.org/simple/wheels/nightly"
priority = "supplemental"

[tool.poetry.dependencies]
python = ">=3.10,<4.0"
openvino = { version = "*", source = "openvino-nightly" }

[tool.poetry.group.dev]
optional = true

[tool.poetry.group.dev.dependencies]
ruff = ">=0.1.0,<=0.12.8"
bandit = ">=1.7.0,<=1.8.6"
pytest = ">=7.0.0,<=8.3.5"
pytest-xdist = ">=3.4.0,<=3.8.0"
pytest-harvest = ">=1.6.0,<=1.10.0"
pandas = ">=2.0.0,<=2.3.1"
jinja2 = ">=3.0.0,<=3.1.6"

[tool.poetry.group.benchmark]
optional = true

[tool.poetry.group.benchmark.dependencies]
pandas = ">=2.0.0,<=2.3.1"
seaborn = ">=0.10.0,<=0.13.2"
tqdm = ">=4.50.0,<=4.67.1"

[tool.poetry.group.fuzzing]
optional = true

[tool.poetry.group.fuzzing.dependencies]
atheris = ">=2.0.0,<=2.3.0"