Skip to content

Parse metadata fields from sdist artifacts #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 14, 2024
Merged

Parse metadata fields from sdist artifacts #7

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a9de409
Parse the PKG-INFO file from sdist archives.
amjith Nov 14, 2024
cb424e2
Add tests for sdist tar.
amjith Nov 14, 2024
a40ae7e
Add tests for sdist zip.
amjith Nov 14, 2024
2121ba3
Find first PKG-INFO and update tests to abstract the contents.
amjith Nov 14, 2024
e69ee20
Fix formatting.
amjith Nov 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 23 additions & 10 deletions metadata_please/sdist.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,17 @@ def from_zip_sdist(zf: ZipFile) -> bytes:
def basic_metadata_from_zip_sdist(zf: ZipFile) -> BasicMetadata:
requires = [f for f in zf.namelist() if f.endswith("/requires.txt")]
requires.sort(key=len)
if not requires:
return BasicMetadata((), frozenset(), "-")
if requires:
requires_data = zf.read(requires[0])
assert requires_data is not None
else:
requires_data = b""

data = zf.read(requires[0])
assert data is not None
return BasicMetadata.from_sdist_pkg_info_and_requires(b"", data)
pkg_info = next(f for f in zf.namelist() if f.endswith("/PKG-INFO"))
Copy link
Member

@thatch thatch Nov 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider sorting this by length and taking the shortest; these might exist in test data and these archives are generally alphabetic order. I'll try to find you a real-world artifact that needs this.

Copy link
Member

@thatch thatch Nov 14, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nose 1.3.1 (test data)
ucxdsa 2024.4.23 (project rename, didn't clean)
relaxed-poetry 0.2.2 (test data)
wish 1.0.1 (.lib dir)

additionally looks like rust projects don't have a top level dir

orjson 3.5.0
rfernet 0.1.3

pkg_info_data = zf.read(pkg_info)
assert pkg_info_data is not None

return BasicMetadata.from_sdist_pkg_info_and_requires(pkg_info_data, requires_data)


def from_tar_sdist(tf: TarFile) -> bytes:
Expand All @@ -58,17 +63,25 @@ def from_tar_sdist(tf: TarFile) -> bytes:
buf.append(f"Requires-Dist: {req}\n")
for extra in sorted(extras):
buf.append(f"Provides-Extra: {extra}\n")

return ("".join(buf)).encode("utf-8")


def basic_metadata_from_tar_sdist(tf: TarFile) -> BasicMetadata:
# XXX Why do ZipFile and TarFile not have a common interface ?!
requires = [f for f in tf.getnames() if f.endswith("/requires.txt")]
requires.sort(key=len)
if not requires:
return BasicMetadata((), frozenset())
if requires:
requires_fo = tf.extractfile(requires[0])
assert requires_fo is not None
else:
requires_fo = b""
Copy link
Member

@thatch thatch Nov 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this looks like a copypaste error from the zip one, I like requires_DATA = b"" and doing the read a couple lines above rather than in the last line of this func.


fo = tf.extractfile(requires[0])
assert fo is not None
pkg_info = next(f for f in tf.getnames() if f.endswith("/PKG-INFO"))

pkg_info_fo = tf.extractfile(pkg_info)
assert pkg_info_fo is not None

return BasicMetadata.from_sdist_pkg_info_and_requires(b"", fo.read())
return BasicMetadata.from_sdist_pkg_info_and_requires(
pkg_info_fo.read(), requires_fo and requires_fo.read()
)
40 changes: 38 additions & 2 deletions metadata_please/tests/sdist.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ def test_requires_as_expected(self) -> None:
z = MemoryZipFile(
{
"foo/__init__.py": b"",
"foo.egg-info/PKG-INFO": b"\n",
"foo.egg-info/requires.txt": b"""\
a
[e]
Expand All @@ -36,6 +37,7 @@ def test_basic_metadata(self) -> None:
z = MemoryZipFile(
{
"foo/__init__.py": b"",
"foo.egg-info/PKG-INFO": b"\n",
"foo.egg-info/requires.txt": b"""\
a
[e]
Expand Down Expand Up @@ -68,6 +70,7 @@ def test_basic_metadata_absl_py_09(self) -> None:
z = MemoryZipFile(
{
"foo/__init__.py": b"",
"foo.egg-info/PKG-INFO": b"\n",
"foo.egg-info/requires.txt": b"""\
six

Expand All @@ -90,11 +93,29 @@ def test_basic_metadata_absl_py_09(self) -> None:
)
self.assertEqual({"test"}, bm.provides_extra)

def test_basic_metadata_fields(self) -> None:
Copy link
Member

@thatch thatch Nov 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you include a couple more words about who does this -- is this from modern setuptools, or something else like flit/poetry/hatchling

Copy link
Contributor Author

@amjith amjith Nov 14, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Update test to have Description.

Move the contents of the PKG-INFO to a module const.

z = MemoryZipFile(
{
"foo/__init__.py": b"",
"foo.egg-info/PKG-INFO": b"Requires-Dist: foo\nVersion: 1.2.58\nSummary: Some Summary\nHome-page: http://example.com\nAuthor: Chicken\nAuthor-email: [email protected]\nKeywords: farm,animals\nRequires-Python: >=3.6\nDescription-Content-Type: text/markdown",
}
)
bm = basic_metadata_from_zip_sdist(z) # type: ignore
self.assertEqual(["foo"], bm.reqs)
self.assertEqual("1.2.58", bm.version)
self.assertEqual("Some Summary", bm.summary)
self.assertEqual("http://example.com", bm.url)
self.assertEqual("Chicken", bm.author)
self.assertEqual("[email protected]", bm.author_email)
self.assertEqual("farm,animals", bm.keywords)
self.assertEqual("text/markdown", bm.long_description_content_type)
self.assertEqual(None, bm.description)


class TarSdistTest(unittest.TestCase):
def test_requires_as_expected(self) -> None:
t = MemoryTarFile(
["foo.egg-info/requires.txt", "foo/__init__.py"],
["foo.egg-info/PKG-INFO", "foo.egg-info/requires.txt", "foo/__init__.py"],
read_value=b"""\
a
[e]
Expand All @@ -113,7 +134,7 @@ def test_requires_as_expected(self) -> None:

def test_basic_metadata(self) -> None:
t = MemoryTarFile(
["foo.egg-info/requires.txt", "foo/__init__.py"],
["foo.egg-info/PKG-INFO", "foo.egg-info/requires.txt", "foo/__init__.py"],
read_value=b"""\
a
[e]
Expand All @@ -126,3 +147,18 @@ def test_basic_metadata(self) -> None:
bm.reqs,
)
self.assertEqual({"e"}, bm.provides_extra)

def test_metadata_fields_from_tar_sdist(self) -> None:
t = MemoryTarFile(
["foo.egg-info/PKG-INFO", "foo/__init__.py"],
read_value=b"""Requires-Dist: foo\nVersion: 1.2.58\nSummary: Some Summary\nHome-page: http://example.com\nAuthor: Chicken\nAuthor-email: [email protected]\nKeywords: farm,animals\nRequires-Python: >=3.6\nDescription-Content-Type: text/markdown\n""",
)
bm = basic_metadata_from_tar_sdist(t) # type: ignore
self.assertEqual("1.2.58", bm.version)
self.assertEqual("Some Summary", bm.summary)
self.assertEqual("http://example.com", bm.url)
self.assertEqual("Chicken", bm.author)
self.assertEqual("[email protected]", bm.author_email)
self.assertEqual("farm,animals", bm.keywords)
self.assertEqual("text/markdown", bm.long_description_content_type)
self.assertEqual(None, bm.description)
Loading