|
a |
|
b/tests/utils/test_package.py |
|
|
1 |
import importlib |
|
|
2 |
import subprocess |
|
|
3 |
import sys |
|
|
4 |
import tarfile |
|
|
5 |
import zipfile |
|
|
6 |
|
|
|
7 |
import pytest |
|
|
8 |
|
|
|
9 |
import edsnlp |
|
|
10 |
from edsnlp.package import package |
|
|
11 |
|
|
|
12 |
|
|
|
13 |
def test_blank_package(nlp, tmp_path): |
|
|
14 |
# Missing metadata makes poetry fail due to missing author / description |
|
|
15 |
if not isinstance(nlp, edsnlp.Pipeline): |
|
|
16 |
pytest.skip("Only running for edsnlp.Pipeline") |
|
|
17 |
|
|
|
18 |
package( |
|
|
19 |
pipeline=nlp, |
|
|
20 |
root_dir=tmp_path, |
|
|
21 |
name="test-model-fail", |
|
|
22 |
metadata={}, |
|
|
23 |
project_type="poetry", |
|
|
24 |
) |
|
|
25 |
|
|
|
26 |
nlp.package( |
|
|
27 |
root_dir=tmp_path, |
|
|
28 |
name="test-model", |
|
|
29 |
metadata={ |
|
|
30 |
"description": "A test model", |
|
|
31 |
"authors": "Test Author <test.author@mail.com>", |
|
|
32 |
}, |
|
|
33 |
project_type="poetry", |
|
|
34 |
distributions=["wheel"], |
|
|
35 |
) |
|
|
36 |
assert (tmp_path / "dist").is_dir() |
|
|
37 |
assert (tmp_path / "dist" / "test_model-0.1.0-py3-none-any.whl").is_file() |
|
|
38 |
assert not (tmp_path / "dist" / "test_model-0.1.0.tar.gz").is_file() |
|
|
39 |
|
|
|
40 |
|
|
|
41 |
@pytest.mark.parametrize("package_name", ["my-test-model", None]) |
|
|
42 |
@pytest.mark.parametrize("manager", ["poetry", "setuptools"]) |
|
|
43 |
def test_package_with_files(nlp, tmp_path, package_name, manager): |
|
|
44 |
if not isinstance(nlp, edsnlp.Pipeline): |
|
|
45 |
pytest.skip("Only running for edsnlp.Pipeline") |
|
|
46 |
|
|
|
47 |
nlp.to_disk(tmp_path / "model", exclude=set()) |
|
|
48 |
|
|
|
49 |
((tmp_path / "test_model").mkdir(parents=True)) |
|
|
50 |
(tmp_path / "test_model" / "__init__.py").write_text('print("Hello World!")\n') |
|
|
51 |
(tmp_path / "test_model" / "empty_folder").mkdir() |
|
|
52 |
(tmp_path / "README.md").write_text( |
|
|
53 |
"""\ |
|
|
54 |
<!-- INSERT --> |
|
|
55 |
# Test Model |
|
|
56 |
""" |
|
|
57 |
) |
|
|
58 |
if manager == "poetry": |
|
|
59 |
(tmp_path / "pyproject.toml").write_text( |
|
|
60 |
"""\ |
|
|
61 |
[build-system] |
|
|
62 |
requires = ["poetry-core>=1.0.0"] |
|
|
63 |
build-backend = "poetry.core.masonry.api" |
|
|
64 |
|
|
|
65 |
[tool.poetry] |
|
|
66 |
name = "test-model" |
|
|
67 |
version = "0.0.0" |
|
|
68 |
description = "A test model" |
|
|
69 |
authors = ["Test Author <test.author@mail.com>"] |
|
|
70 |
readme = "README.md" |
|
|
71 |
|
|
|
72 |
[tool.poetry.dependencies] |
|
|
73 |
python = ">=3.7" |
|
|
74 |
build = "*" # sample light package to install |
|
|
75 |
""" |
|
|
76 |
) |
|
|
77 |
elif manager == "setuptools": |
|
|
78 |
(tmp_path / "pyproject.toml").write_text( |
|
|
79 |
"""\ |
|
|
80 |
[build-system] |
|
|
81 |
requires = ["setuptools>=42", "wheel"] |
|
|
82 |
build-backend = "setuptools.build_meta" |
|
|
83 |
|
|
|
84 |
[project] |
|
|
85 |
name = "test-model" |
|
|
86 |
version = "0.0.0" |
|
|
87 |
description = "A test model" |
|
|
88 |
authors = [ |
|
|
89 |
{name = "Test Author", email = "test.author@mail.com"} |
|
|
90 |
] |
|
|
91 |
readme = "README.md" |
|
|
92 |
requires-python = ">=3.7" |
|
|
93 |
|
|
|
94 |
dependencies = [ |
|
|
95 |
"build" |
|
|
96 |
] |
|
|
97 |
""" |
|
|
98 |
) |
|
|
99 |
package( |
|
|
100 |
name=package_name, |
|
|
101 |
pipeline=tmp_path / "model", |
|
|
102 |
root_dir=tmp_path, |
|
|
103 |
check_dependencies=False, |
|
|
104 |
version="0.1.0", |
|
|
105 |
distributions=None, |
|
|
106 |
metadata={ |
|
|
107 |
"description": "A new description", |
|
|
108 |
"authors": "Test Author <test.author@mail.com>", |
|
|
109 |
}, |
|
|
110 |
readme_replacements={ |
|
|
111 |
"<!-- INSERT -->": "Replaced !", |
|
|
112 |
}, |
|
|
113 |
) |
|
|
114 |
|
|
|
115 |
module_name = "test_model" if package_name is None else "my_test_model" |
|
|
116 |
|
|
|
117 |
assert (tmp_path / "dist").is_dir() |
|
|
118 |
assert (tmp_path / "dist" / f"{module_name}-0.1.0.tar.gz").is_file() |
|
|
119 |
assert (tmp_path / "dist" / f"{module_name}-0.1.0-py3-none-any.whl").is_file() |
|
|
120 |
assert (tmp_path / "pyproject.toml").is_file() |
|
|
121 |
|
|
|
122 |
with zipfile.ZipFile( |
|
|
123 |
tmp_path / "dist" / f"{module_name}-0.1.0-py3-none-any.whl" |
|
|
124 |
) as zf: |
|
|
125 |
# check files |
|
|
126 |
assert set(zf.namelist()) == { |
|
|
127 |
f"{module_name}-0.1.0.dist-info/METADATA", |
|
|
128 |
f"{module_name}-0.1.0.dist-info/RECORD", |
|
|
129 |
f"{module_name}-0.1.0.dist-info/WHEEL", |
|
|
130 |
f"{module_name}/__init__.py", |
|
|
131 |
f"{module_name}/artifacts/config.cfg", |
|
|
132 |
f"{module_name}/artifacts/meta.json", |
|
|
133 |
f"{module_name}/artifacts/tokenizer", |
|
|
134 |
"test_model/__init__.py", |
|
|
135 |
} |
|
|
136 |
# check description |
|
|
137 |
with zf.open(f"{module_name}-0.1.0.dist-info/METADATA") as f: |
|
|
138 |
assert b"A new description" in f.read() |
|
|
139 |
|
|
|
140 |
with tarfile.open(tmp_path / "dist" / f"{module_name}-0.1.0.tar.gz") as tf: |
|
|
141 |
# check files |
|
|
142 |
assert set(tf.getnames()) == { |
|
|
143 |
f"{module_name}-0.1.0/PKG-INFO", |
|
|
144 |
f"{module_name}-0.1.0/README.md", |
|
|
145 |
f"{module_name}-0.1.0/artifacts/config.cfg", |
|
|
146 |
f"{module_name}-0.1.0/artifacts/meta.json", |
|
|
147 |
f"{module_name}-0.1.0/artifacts/tokenizer", |
|
|
148 |
f"{module_name}-0.1.0/{module_name}/__init__.py", |
|
|
149 |
f"{module_name}-0.1.0/pyproject.toml", |
|
|
150 |
f"{module_name}-0.1.0/test_model/__init__.py", |
|
|
151 |
} |
|
|
152 |
# check description |
|
|
153 |
with tf.extractfile(f"{module_name}-0.1.0/PKG-INFO") as f: |
|
|
154 |
assert b"A new description" in f.read() |
|
|
155 |
|
|
|
156 |
with tf.extractfile(f"{module_name}-0.1.0/README.md") as f: |
|
|
157 |
assert b"Replaced !" in f.read() |
|
|
158 |
|
|
|
159 |
# pip install the whl file |
|
|
160 |
(tmp_path / "site-packages").mkdir(exist_ok=True) |
|
|
161 |
subprocess.check_output( |
|
|
162 |
[ |
|
|
163 |
sys.executable, |
|
|
164 |
"-m", |
|
|
165 |
"pip", |
|
|
166 |
"install", |
|
|
167 |
"-vvv", |
|
|
168 |
"--target", |
|
|
169 |
str(tmp_path / "site-packages"), |
|
|
170 |
str(tmp_path / "dist" / f"{module_name}-0.1.0-py3-none-any.whl"), |
|
|
171 |
], |
|
|
172 |
stderr=subprocess.STDOUT, |
|
|
173 |
) |
|
|
174 |
|
|
|
175 |
site_packages = tmp_path / "site-packages" |
|
|
176 |
sys.path.insert(0, str(site_packages)) |
|
|
177 |
# check site-package files |
|
|
178 |
files = {str(f.relative_to(site_packages)) for f in set(site_packages.rglob("*"))} |
|
|
179 |
assert files >= { |
|
|
180 |
f"{module_name}/artifacts", |
|
|
181 |
f"{module_name}/artifacts/config.cfg", |
|
|
182 |
f"{module_name}/artifacts/meta.json", |
|
|
183 |
f"{module_name}/artifacts/tokenizer", |
|
|
184 |
} |
|
|
185 |
|
|
|
186 |
module = importlib.import_module(module_name) |
|
|
187 |
|
|
|
188 |
with open(module.__file__) as f: |
|
|
189 |
assert f.read() == ( |
|
|
190 |
('print("Hello World!")\n' if package_name is None else "") |
|
|
191 |
+ """ |
|
|
192 |
# ----------------------------------------- |
|
|
193 |
# This section was autogenerated by edsnlp |
|
|
194 |
# ----------------------------------------- |
|
|
195 |
|
|
|
196 |
import edsnlp |
|
|
197 |
from pathlib import Path |
|
|
198 |
from typing import Optional, Dict, Any |
|
|
199 |
|
|
|
200 |
__version__ = '0.1.0' |
|
|
201 |
|
|
|
202 |
def load( |
|
|
203 |
overrides: Optional[Dict[str, Any]] = None, |
|
|
204 |
) -> edsnlp.Pipeline: |
|
|
205 |
path_outside = Path(__file__).parent / "../artifacts" |
|
|
206 |
path_inside = Path(__file__).parent / "artifacts" |
|
|
207 |
path = path_inside if path_inside.exists() else path_outside |
|
|
208 |
model = edsnlp.load(path, overrides=overrides) |
|
|
209 |
return model |
|
|
210 |
""" |
|
|
211 |
) |
|
|
212 |
module.load() |
|
|
213 |
edsnlp.load(module_name) |