diff options
| -rw-r--r-- | CHANGELOG.md | 35 | ||||
| -rw-r--r-- | debian/changelog | 7 | ||||
| -rw-r--r-- | debian/control | 2 | ||||
| -rw-r--r-- | docs/api/wn.rst | 44 | ||||
| -rw-r--r-- | docs/cli.rst | 34 | ||||
| -rw-r--r-- | docs/setup.rst | 17 | ||||
| -rw-r--r-- | pyproject.toml | 2 | ||||
| -rw-r--r-- | tests/_config_test.py | 10 | ||||
| -rw-r--r-- | tests/_util_test.py | 24 | ||||
| -rw-r--r-- | tests/secondary_query_test.py | 29 | ||||
| -rw-r--r-- | wn/__init__.py | 3 | ||||
| -rw-r--r-- | wn/__main__.py | 45 | ||||
| -rw-r--r-- | wn/_add.py | 8 | ||||
| -rw-r--r-- | wn/_config.py | 206 | ||||
| -rw-r--r-- | wn/_core.py | 5 | ||||
| -rw-r--r-- | wn/_db.py | 10 | ||||
| -rw-r--r-- | wn/_module_functions.py | 6 | ||||
| -rw-r--r-- | wn/_util.py | 6 | ||||
| -rw-r--r-- | wn/constants.py | 5 | ||||
| -rw-r--r-- | wn/project.py | 7 |
20 files changed, 426 insertions, 79 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d258c3..cbc4983 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,40 @@ ## [Unreleased][unreleased] + +## [v1.1.0] + +**Release date: 2026-03-21** + +### Added + +* `cache` subcommand ([#313]) +* `wn.config.list_cache_entries()` method ([#313]) +* Support for `WN_DATA_DIR` environment variable ([#314]) + +### Changed + +* The schema hashing function is now resilient to ordering and SQL DB + operations ([#319]) + +### Fixed + +* `Synset.translate()` resets the lexicon configuration of the + translated sysets ([#316]); `Sense.translate()` and + `Word.translate()` derive from `Synset.translate()` so nothing + special needs to be done for them. + + +## [v1.0.0] + +**Release date: 2026-01-31** + Notable changes in this release: * A new version of the database schema requires a database rebuild * A new `wn.ili` module deals with ILI files and objects; interlingual queries still use the `Synset.ili` member, which is now a simple `str` * The Open English Wordnet versions 2025 and 2025+ are added to the index +* The Open Multilingual Wordnet 2.0 is added to the index ### Index @@ -799,6 +828,8 @@ the https://github.com/nltk/wordnet/ code which had been effectively abandoned, but this is an entirely new codebase. +[v1.1.0]: ../../releases/tag/v1.1.0 +[v1.0.0]: ../../releases/tag/v1.0.0 [v0.14.0]: ../../releases/tag/v0.14.0 [v0.13.0]: ../../releases/tag/v0.13.0 [v0.12.0]: ../../releases/tag/v0.12.0 @@ -945,3 +976,7 @@ abandoned, but this is an entirely new codebase. [#301]: https://github.com/goodmami/wn/issues/301 [#302]: https://github.com/goodmami/wn/issues/302 [#303]: https://github.com/goodmami/wn/issues/303 +[#313]: https://github.com/goodmami/wn/issues/313 +[#314]: https://github.com/goodmami/wn/issues/314 +[#316]: https://github.com/goodmami/wn/issues/316 +[#319]: https://github.com/goodmami/wn/issues/319 diff --git a/debian/changelog b/debian/changelog index 93e145d..7fe5469 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +python-wn (1.1.0-1) unstable; urgency=medium + + * New upstream release. + * d/control: Update Standards-Version to 4.7.4. + + -- Danial Behzadi <dani.behzi@ubuntu.com> Thu, 02 Apr 2026 02:11:50 +0330 + python-wn (1.0.0-3) unstable; urgency=medium * Team Upload diff --git a/debian/control b/debian/control index 1d04e46..6dd5f11 100644 --- a/debian/control +++ b/debian/control @@ -12,7 +12,7 @@ Build-Depends: python3-httpx, python3-setuptools, Testsuite: autopkgtest-pkg-python -Standards-Version: 4.7.3 +Standards-Version: 4.7.4 Homepage: https://github.com/goodmami/wn Vcs-Git: https://salsa.debian.org/python-team/packages/python-wn.git Vcs-Browser: https://salsa.debian.org/python-team/packages/python-wn diff --git a/docs/api/wn.rst b/docs/api/wn.rst index 37750c6..3d4a50f 100644 --- a/docs/api/wn.rst +++ b/docs/api/wn.rst @@ -492,10 +492,54 @@ directly. Configuration should occur through the single .. automethod:: add_project_version .. automethod:: get_project_info .. automethod:: get_cache_path + .. automethod:: list_cache_entries .. automethod:: update .. automethod:: load_index +Auxiliary WNConfig Types +'''''''''''''''''''''''' + +The following classes are argument or return types of +:class:`~wn._config.WNConfig` objects. They are documented here for reference, +but are not meant to be created directly. + +.. autoclass:: wn._config.ResourceType + + Enumeration of resource types. + + .. autoattribute:: WORDNET + .. autoattribute:: ILI + + +.. autoclass:: wn._config.ProjectInfo + :members: + :undoc-members: + + Dictionary of information about a project. + + +.. autoclass:: wn._config.VersionInfo + :members: + :undoc-members: + + Dictionary of information about a resource version. + + +.. autoclass:: wn._config.ResolvedProjectInfo + :members: + :undoc-members: + + Dictionary of information about a specific project resource. + + +.. autoclass:: wn._config.CacheEntry + :members: + :undoc-members: + + Dictionary of information about files in the download cache. + + Exceptions ---------- diff --git a/docs/cli.rst b/docs/cli.rst index 4c490a3..5ad5b7e 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -40,6 +40,38 @@ specifiers or URLs. database. +cache +----- + +View the files in the download cache. The ``download`` command caches the (often +compressed) files to the filesystem prior to adding to Wn's database. The files +are renamed with a hash of the URL to avoid name clashes, but this also makes it +hard to determine what a particular file is. This command cross-references the +downloaded files with what is in the index. An optional project specifier +argument can help narrow down the results. + +.. code-block:: console + + $ python -m wn cache # many results; abbreviated here + af909070c29845b952d1799551bffc302e28d2c5 own-en 1.0.0 https://github.com/own-pt/openWordnet-PT/releases/download/v1.0.0/own-en.tar.gz + e25af66e46775b00d689619787013e6a35e5cbf7 oewn 2025 https://en-word.net/static/english-wordnet-2025.xml.gz + 5a26d97a0081996db4cd621638a8a9b0da09aa25 odenet 1.4 https://github.com/hdaSprachtechnologie/odenet/releases/download/v1.4/odenet-1.4.tar.xz + [...] + $ python -m wn cache "oewn:2025*" # narrowed results + e25af66e46775b00d689619787013e6a35e5cbf7 oewn 2025 https://en-word.net/static/english-wordnet-2025.xml.gz + 0f5371187dcfe7e05f2a93ab85b4e1168859a5c2 oewn 2025+ https://en-word.net/static/english-wordnet-2025-plus.xml.gz + +.. option:: --full-paths-only + + Only print the full path of each cache file. This can be useful when one + wants to pipe the results to other commands. For example, on Unix-like + systems, the following will delete matching cache entries: + + .. code-block:: console + + $ python -m wn cache --full-paths-only "omw*:1.4" | xargs rm + + lexicons -------- @@ -118,7 +150,7 @@ problems and print a report. .. code-block:: - $ python -m wn validate --select E W201 W204 deWordNet.xml + $ python -m wn validate --select E,W201,W204 deWordNet.xml .. option:: --output-file FILE diff --git a/docs/setup.rst b/docs/setup.rst index cd5f747..5602d0d 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -53,12 +53,27 @@ change the directory Wn uses for storing data locally, modify the import wn wn.config.data_directory = '~/Projects/wn_data' +You can alternatively set the ``WN_DATA_DIR`` environment variable +prior to importing Wn. On Unix-like systems, this would look like: + +.. code-block:: console + + $ export WN_DATA_DIR=~/path/to/wn_data + $ python3 ... + +If you are using Wn from the command line, a third option is to use +the ``--dir`` or ``-d`` option: + +.. code-block:: console + + $ python3 -m wn --dir ~/path/to/wn_data download ... + There are some things to note: - The downloads directory and database path are always relative to the data directory and cannot be changed directly. - This change only affects subsequent operations, so any data in the - previous location will not be moved nor deleted. + previous location will not be moved or deleted. - This change only affects the current session. If you want a script or application to always use the new location, it must reset the data directory each time it is initialized. diff --git a/pyproject.toml b/pyproject.toml index ecbf1f9..3305546 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ requires-python = ">=3.10" license = {file = "LICENSE"} keywords = ["wordnet", "interlingual", "linguistics", "language", "library"] authors = [ - {name = "Michael Wayne Goodman", email = "goodman.m.w@gmail.com"} + {name = "Michael Wayne Goodman", email = "1428419+goodmami@users.noreply.github.com"} ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/tests/_config_test.py b/tests/_config_test.py new file mode 100644 index 0000000..33e0885 --- /dev/null +++ b/tests/_config_test.py @@ -0,0 +1,10 @@ +from pathlib import Path + +from wn._config import WNConfig + + +def test_envvar_data_dir(monkeypatch, tmp_path): + assert WNConfig().data_directory == Path.home() / ".wn_data" + with monkeypatch.context() as mp: + mp.setenv("WN_DATA_DIR", str(tmp_path)) + assert WNConfig().data_directory == tmp_path diff --git a/tests/_util_test.py b/tests/_util_test.py index 9e9290e..9155b39 100644 --- a/tests/_util_test.py +++ b/tests/_util_test.py @@ -1,4 +1,10 @@ -from wn._util import flatten, normalize_form, unique_list +from wn._util import ( + flatten, + format_lexicon_specifier, + normalize_form, + split_lexicon_specifier, + unique_list, +) def test_flatten(): @@ -28,3 +34,19 @@ def test_normalize_form(): assert normalize_form("so\xf1ar") == "sonar" # soñar with single ñ character assert normalize_form("son\u0303ar") == "sonar" # soñar with combining tilde assert normalize_form("Weiß") == "weiss" + + +def test_format_lexicon_specifier(): + assert format_lexicon_specifier("", "") == ":" + assert format_lexicon_specifier("foo", "") == "foo:" + assert format_lexicon_specifier("", "bar") == ":bar" + assert format_lexicon_specifier("foo", "bar") == "foo:bar" + + +def test_split_lexicon_specifier(): + assert split_lexicon_specifier("") == ("", "") + assert split_lexicon_specifier(":") == ("", "") + assert split_lexicon_specifier("foo") == ("foo", "") + assert split_lexicon_specifier("foo:") == ("foo", "") + assert split_lexicon_specifier(":bar") == ("", "bar") + assert split_lexicon_specifier("foo:bar") == ("foo", "bar") diff --git a/tests/secondary_query_test.py b/tests/secondary_query_test.py index ee7763b..5af1705 100644 --- a/tests/secondary_query_test.py +++ b/tests/secondary_query_test.py @@ -21,6 +21,16 @@ def test_word_translate(): assert len(wn.word("test-es-ejemplo-n").translate(lang="en")) == 1 +@pytest.mark.usefixtures("mini_db") +def test_word_translate_issue_316(): + # https://github.com/goodmami/wn/issues/316 + es = wn.Wordnet("test-es") + es_w = es.word("test-es-información-n") + translations = es_w.translate(lexicon="test-en") + assert len(translations) == 1 + assert next(iter(translations.values()))[0].forms() == ["information"] + + @pytest.mark.usefixtures("mini_db_1_1") def test_word_lemma_tags(): en = wn.Wordnet("test-en") @@ -118,6 +128,14 @@ def test_sense_translate(): @pytest.mark.usefixtures("mini_db") +def test_sense_translate_issue_316(): + # https://github.com/goodmami/wn/issues/316 + es = wn.Wordnet("test-es") + es_s = es.sense("test-es-información-n-0001-01") + assert es_s.translate(lexicon="test-en")[0].counts() == [3] + + +@pytest.mark.usefixtures("mini_db") def test_synset_senses(): assert len(wn.synset("test-en-0003-v").senses()) == 2 assert len(wn.synset("test-es-0003-v").senses()) == 2 @@ -179,6 +197,17 @@ def test_synset_translate(): assert len(wn.synset("test-es-0001-n").translate(lang="en")) == 1 +@pytest.mark.usefixtures("mini_db") +def test_synset_translate_issue_316(): + # https://github.com/goodmami/wn/issues/316 + es = wn.Wordnet("test-es") + en = wn.Wordnet("test-en") + es_ss = es.synset("test-es-0001-n") + en_ss = en.synset("test-en-0001-n") + assert es_ss.translate(lexicon="test-en")[0].definition() == en_ss.definition() + assert en_ss.translate(lexicon="test-es")[0].definition() == es_ss.definition() + + @pytest.mark.usefixtures("uninitialized_datadir") def test_word_sense_order(datadir): wn.add(datadir / "sense-member-order.xml") diff --git a/wn/__init__.py b/wn/__init__.py index 05d162c..5e985d2 100644 --- a/wn/__init__.py +++ b/wn/__init__.py @@ -37,6 +37,7 @@ __all__ = ( "word", "words", ) +__version__ = "1.1.0" from wn._add import add, add_lexical_resource, remove from wn._config import config # noqa: F401 @@ -75,5 +76,3 @@ from wn._module_functions import ( words, ) from wn._wordnet import Wordnet - -__version__ = "1.0.0" diff --git a/wn/__main__.py b/wn/__main__.py index 8874fcf..b63ed5c 100644 --- a/wn/__main__.py +++ b/wn/__main__.py @@ -11,19 +11,38 @@ from wn.project import iterpackages from wn.validate import validate -def _download(args): +def _download(args: argparse.Namespace) -> None: if args.index: wn.config.load_index(args.index) for target in args.target: wn.download(target, add=args.add) -def _lexicons(args): +def _cache(args: argparse.Namespace) -> None: + cache_entries = wn.config.list_cache_entries(args.ARG) + if args.full_paths_only: + for cache_entry in cache_entries: + print(str(cache_entry["path"])) + else: + for cache_entry in cache_entries: + print( + "\t".join( + [ + str(cache_entry["path"].name), + cache_entry["id"] or "?", + cache_entry["version"] or "?", + cache_entry["url"] or "?", + ] + ) + ) + + +def _lexicons(args: argparse.Namespace) -> None: for lex in wn.lexicons(lang=args.lang, lexicon=args.lexicon): print("\t".join((lex.id, lex.version, f"[{lex.language}]", lex.label))) -def _projects(args): +def _projects(args: argparse.Namespace) -> None: for info in wn.projects(): key = "i" key += "c" if info["cache"] else "-" @@ -41,7 +60,7 @@ def _projects(args): ) -def _validate(args): +def _validate(args: argparse.Namespace) -> None: all_valid = True selectseq = [check.strip() for check in args.select.split(",")] for package in iterpackages(args.FILE): @@ -125,6 +144,24 @@ parser_download.add_argument( parser_download.set_defaults(func=_download) +parser_cache = sub_parsers.add_parser( + "cache", + description="View Wn's download cache.", + help="view the download cache", +) +parser_cache.add_argument( + "ARG", + help="project specifier or URL", + nargs="?", + default="*", +) +parser_cache.add_argument( + "--full-paths-only", + action="store_true", + help="print the full paths of cache entries without other data", +) +parser_cache.set_defaults(func=_cache) + parser_lexicons = sub_parsers.add_parser( "lexicons", description="Display a list of installed lexicons.", @@ -9,9 +9,9 @@ from itertools import islice from pathlib import Path from typing import TypeVar, cast -from wn import constants, lmf from wn import ili as _ili -from wn._config import config +from wn import lmf +from wn._config import ResourceType, config from wn._db import connect from wn._exceptions import Error from wn._queries import ( @@ -108,9 +108,9 @@ def add( try: for package in iterpackages(source): match package.type: - case constants._WORDNET: + case ResourceType.WORDNET: _add_lmf(package.resource_file(), progress, progress_handler) - case constants._ILI: + case ResourceType.ILI: _add_ili(package.resource_file(), progress) case _: raise Error(f"unknown package type: {package.type}") diff --git a/wn/_config.py b/wn/_config.py index e8eb049..d058c85 100644 --- a/wn/_config.py +++ b/wn/_config.py @@ -2,10 +2,13 @@ Local configuration settings. """ +import os from collections.abc import Sequence +from enum import Enum +from fnmatch import fnmatch from importlib.resources import as_file, files from pathlib import Path -from typing import Any +from typing import Any, TypedDict try: # python_version >= 3.11 @@ -15,20 +18,64 @@ except ImportError: from wn._exceptions import ConfigurationError, ProjectError from wn._types import AnyPath -from wn._util import format_lexicon_specifier, short_hash, split_lexicon_specifier -from wn.constants import _WORDNET +from wn._util import ( + format_lexicon_specifier, + is_str_key_dict, + short_hash, + split_lexicon_specifier, +) # The index file is a project file of Wn with as_file(files("wn") / "index.toml") as index_file: INDEX_FILE_PATH = index_file # The directory where downloaded and added data will be stored. -DEFAULT_DATA_DIRECTORY = Path.home() / ".wn_data" +DEFAULT_DATA_DIR = Path.home() / ".wn_data" DATABASE_FILENAME = "wn.db" +class ResourceType(str, Enum): + WORDNET = "wordnet" + ILI = "ili" + + +class VersionInfo(TypedDict): + resource_urls: list[str] + license: str | None + error: str | None + + +class ProjectInfo(TypedDict): + type: ResourceType + label: str | None + language: str | None + license: str | None + error: str | None + versions: dict[str, VersionInfo] + + +class ResolvedProjectInfo(TypedDict): + id: str + version: str + type: ResourceType + label: str | None + language: str | None + license: str | None + resource_urls: list[str] + cache: Path | None + + +class CacheEntry(TypedDict): + path: Path + id: str | None + version: str | None + url: str | None + + class WNConfig: + _projects: dict[str, ProjectInfo] + def __init__(self): - self._data_directory = DEFAULT_DATA_DIRECTORY + self._data_directory = Path(os.getenv("WN_DATA_DIR", default=DEFAULT_DATA_DIR)) self._projects = {} self._dbpath = self._data_directory / DATABASE_FILENAME self.allow_multithreading = False @@ -52,7 +99,7 @@ class WNConfig: return dir @data_directory.setter - def data_directory(self, path): + def data_directory(self, path: AnyPath) -> None: dir = Path(path).expanduser() if dir.exists() and not dir.is_dir(): raise ConfigurationError(f"path exists and is not a directory: {dir}") @@ -82,14 +129,14 @@ class WNConfig: return dir @property - def index(self) -> dict[str, dict]: + def index(self) -> dict[str, ProjectInfo]: """The project index.""" return self._projects def add_project( self, id: str, - type: str = _WORDNET, + type: ResourceType = ResourceType.WORDNET, label: str | None = None, language: str | None = None, license: str | None = None, @@ -110,15 +157,14 @@ class WNConfig: """ if id in self._projects: raise ValueError(f"project already added: {id}") - self._projects[id] = { - "type": type, - "label": label, - "language": language, - "versions": {}, - "license": license, - } - if error: - self._projects[id]["error"] = error + self._projects[id] = ProjectInfo( + type=ResourceType(type), + label=label, + language=language, + license=license, + error=error, + versions={}, + ) def add_project_version( self, @@ -142,22 +188,19 @@ class WNConfig: is accessed """ - version_data: dict[str, Any] - if url and not error: - version_data = {"resource_urls": url.split()} - elif error and not url: - version_data = {"error": error} - elif url and error: + if url and error: spec = format_lexicon_specifier(id, version) raise ConfigurationError(f"{spec} specifies both url and redirect") - else: - version_data = {} - if license: - version_data["license"] = license + + version_data = VersionInfo( + resource_urls=url.split() if (url and not error) else [], + license=license, + error=error, + ) project = self._projects[id] project["versions"][version] = version_data - def get_project_info(self, arg: str) -> dict: + def get_project_info(self, arg: str) -> ResolvedProjectInfo: """Return information about an indexed project version. If the project has been downloaded and cached, the ``"cache"`` @@ -177,8 +220,8 @@ class WNConfig: id, version = split_lexicon_specifier(arg) if id not in self._projects: raise ProjectError(f"no such project id: {id}") - project: dict = self._projects[id] - if "error" in project: + project: ProjectInfo = self._projects[id] + if project["error"]: raise ProjectError(project["error"]) versions: dict = project["versions"] @@ -189,33 +232,69 @@ class WNConfig: elif version not in versions: raise ProjectError(f"no such version: {version!r} ({id})") info = versions[version] - if "error" in info: + if info["error"]: raise ProjectError(info["error"]) urls = info.get("resource_urls", []) - return { - "id": id, - "version": version, - "type": project["type"], - "label": project["label"], - "language": project["language"], - "license": info.get("license", project.get("license")), - "resource_urls": urls, - "cache": _get_cache_path_for_urls(self, urls), - } + return ResolvedProjectInfo( + id=id, + version=version, + type=project["type"], + label=project["label"], + language=project["language"], + license=info.get("license", project.get("license")), + resource_urls=urls, + cache=_get_cache_path_for_urls(self, urls), + ) def get_cache_path(self, url: str) -> Path: """Return the path for caching *url*. - Note that in general this is just a path operation and does + Note that this is just a path operation and does not signify that the file exists in the file system. """ filename = short_hash(url) return self.downloads_directory / filename - def update(self, data: dict) -> None: + def list_cache_entries(self, arg: str = "*") -> list[CacheEntry]: + """Return a list of cached resources. + + Use *arg* as a pattern to match project specifiers. It + defaults to `"*"` to select all cached entries. + + Each entry on the list is a dictionary with the keys: + * `"path"` -- the path of the cached file + * `"id"` -- the ID of the cached resource + * `"version"` -- the version of the cached resource + * `"url"` -- the URL of the cached resource + + Note that cached files are stored with a hash of their URL as + the filename and that it is not feasible to recover the URL + from the hash alone. Therefore, for lexicons downloaded with a + URL that does not appear in the index, the ID, version, and URL + and will be :python:`None` instead. + """ + arg = arg.strip() + cache_map = _cache_map(self) + entries: list[CacheEntry] = [] + for cache_path in self.downloads_directory.iterdir(): + if cache_path in cache_map: + id, version, url = cache_map[cache_path] + specifier = format_lexicon_specifier(id, version) + if not (fnmatch(specifier, arg) or url == arg): + continue + entries.append( + CacheEntry(path=cache_path, id=id, version=version, url=url) + ) + elif arg in ("*", "*:*"): + entries.append( + CacheEntry(path=cache_path, id=None, version=None, url=None) + ) + return entries + + def update(self, data: dict[str, Any]) -> None: """Update the configuration with items in *data*. Items are only inserted or replaced, not deleted. If a project @@ -225,9 +304,22 @@ class WNConfig: to the indexed project. """ - if "data_directory" in data: - self.data_directory = data["data_directory"] - for id, project in data.get("index", {}).items(): + if datadir := data.get("data_directory"): + if not isinstance(datadir, (str, Path)): + raise ConfigurationError( + "data_directory must be a str or Path, " + f"not {type(datadir).__name__}" + ) + self.data_directory = datadir + if index := data.get("index", {}): + if not is_str_key_dict(index): + raise ConfigurationError("index must be a dict with str keys") + self._update_index(index) + + def _update_index(self, index: dict[str, Any]) -> None: + for id, project in index.items(): + if not is_str_key_dict(project): + raise ConfigurationError(f"invalid project: {project}") if id in self._projects: # validate that they are the same _project = self._projects[id] @@ -237,14 +329,14 @@ class WNConfig: else: self.add_project( id, - type=project.get("type", _WORDNET), + type=project.get("type", ResourceType.WORDNET), label=project.get("label"), language=project.get("language"), license=project.get("license"), error=project.get("error"), ) for version, info in project.get("versions", {}).items(): - if "url" in info and "error" in project: + if info.get("url") and project.get("error"): spec = format_lexicon_specifier(id, version) raise ConfigurationError(f"{spec} url specified with default error") self.add_project_version( @@ -281,6 +373,8 @@ class WNConfig: index = tomllib.load(indexfile) except tomllib.TOMLDecodeError as exc: raise ConfigurationError("malformed index file") from exc + if not is_str_key_dict(index): + raise ConfigurationError("invalid index file") self.update({"index": index}) @@ -295,5 +389,21 @@ def _get_cache_path_for_urls( return None +def _cache_map(config: WNConfig) -> dict[Path, tuple[str, str, str]]: + """Return a dict of cache hashes to resource info tuples. + + Each tuple contains the id, version, and URL of the indexed + resource. The hash is based on the URL and the tuple only contains + information from the index. They do not indicate whether the + resource has been cached. + """ + return { + config.get_cache_path(url): (id, version, url) + for id, p_info in config.index.items() + for version, v_info in p_info["versions"].items() + for url in v_info["resource_urls"] + } + + config = WNConfig() config.load_index(INDEX_FILE_PATH) diff --git a/wn/_core.py b/wn/_core.py index fdf62c0..5fecaa3 100644 --- a/wn/_core.py +++ b/wn/_core.py @@ -946,8 +946,11 @@ class Synset(_Relatable): if not ili: return [] lexicons = resolve_lexicon_specifiers(lexicon=(lexicon or "*"), lang=lang) + lexconf = LexiconConfiguration( + lexicons=tuple(lexicons), expands=(), default_mode=False + ) return [ - Synset(*data, _lexconf=self._lexconf) + Synset(*data, _lexconf=lexconf) for data in get_synsets_for_ilis((ili,), lexicons) ] @@ -34,7 +34,7 @@ DEBUG = False # >>> wn._db.schema_hash(conn) # COMPATIBLE_SCHEMA_HASHES = { - "8348fc1a6254f514294a1dc70458e0733742935d", + "f439c9bd27f809f64ee42896fb0fc20c5d00fd99", } @@ -144,7 +144,13 @@ def list_lexicons_safe(conn: sqlite3.Connection | None = None) -> list[str]: def schema_hash(conn: sqlite3.Connection) -> str: - query = "SELECT sql FROM sqlite_master WHERE NOT sql ISNULL" + query = """ + SELECT sql + FROM sqlite_schema + WHERE NOT sql ISNULL + AND name NOT LIKE 'sqlite_stat%' + ORDER BY sql ASC + """ schema = "\n\n".join(row[0] for row in conn.execute(query)) return short_hash(schema) diff --git a/wn/_module_functions.py b/wn/_module_functions.py index 1f6f362..f2beac3 100644 --- a/wn/_module_functions.py +++ b/wn/_module_functions.py @@ -1,6 +1,6 @@ from typing import Literal, overload -from wn._config import config +from wn._config import ResolvedProjectInfo, config from wn._core import Form, Sense, Synset, Word from wn._db import clear_connections, connect, list_lexicons_safe from wn._download import download @@ -10,7 +10,7 @@ from wn._util import format_lexicon_specifier from wn._wordnet import Wordnet -def projects() -> list[dict]: +def projects() -> list[ResolvedProjectInfo]: """Return the list of indexed projects. This returns the same dictionaries of information as @@ -32,7 +32,7 @@ def projects() -> list[dict]: config.get_project_info(format_lexicon_specifier(project_id, version)) for project_id, project_info in index.items() for version in project_info.get("versions", []) - if "resource_urls" in project_info["versions"][version] + if not project_info["versions"][version]["error"] ] diff --git a/wn/_util.py b/wn/_util.py index 7d89cd2..de9feda 100644 --- a/wn/_util.py +++ b/wn/_util.py @@ -3,7 +3,7 @@ import hashlib from collections.abc import Hashable, Iterable from pathlib import Path -from typing import TypeVar +from typing import Any, TypeGuard, TypeVar from unicodedata import combining, normalize from wn._types import VersionInfo @@ -34,6 +34,10 @@ def is_xml(path: Path) -> bool: return _inspect_file_signature(path, b"<?xml ") +def is_str_key_dict(obj: Any) -> TypeGuard[dict[str, Any]]: + return isinstance(obj, dict) and all(isinstance(key, str) for key in obj) + + def _inspect_file_signature(path: Path, signature: bytes) -> bool: if path.is_file(): with path.open("rb") as f: diff --git a/wn/constants.py b/wn/constants.py index 32b0da8..01a3ae8 100644 --- a/wn/constants.py +++ b/wn/constants.py @@ -342,8 +342,3 @@ LEXICOGRAPHER_FILES = { "verb.weather": 43, "adj.ppl": 44, } - -# resource types - -_WORDNET = "wordnet" -_ILI = "ili" diff --git a/wn/project.py b/wn/project.py index e69262f..f5c22ad 100644 --- a/wn/project.py +++ b/wn/project.py @@ -11,11 +11,10 @@ from collections.abc import Iterator from pathlib import Path from wn import ili, lmf -from wn._config import config +from wn._config import ResourceType, config from wn._exceptions import Error from wn._types import AnyPath from wn._util import is_gzip, is_lzma -from wn.constants import _ILI, _WORDNET _ADDITIONAL_FILE_SUFFIXES = ("", ".txt", ".md", ".rst") @@ -38,9 +37,9 @@ def _package_directory_types(path: Path) -> list[tuple[Path, str]]: def _resource_file_type(path: Path) -> str | None: if lmf.is_lmf(path): - return _WORDNET + return ResourceType.WORDNET elif ili.is_ili_tsv(path): - return _ILI + return ResourceType.ILI return None |