Skip to content

Improve package assembly functions #4614

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AyanSinhaMahapatra wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Improve package assembly functions #4614

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 12 additions & 19 deletions src/packagedcode/cargo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,19 +66,14 @@ def assemble(cls, package_data, resource, codebase, package_adder):
for attribute in attributes_to_copy:
package_data.extra_data[attribute] = 'workspace'
workspace_package_data[attribute] = getattr(package_data, attribute)

datafile_path_patterns = CargoTomlHandler.path_patterns + CargoLockHandler.path_patterns

workspace_root = resource.parent(codebase)
if not workspace_root:
# If there's no parent (e.g., scanning a single file), use the directory part of the resource path
workspace_root_path = os.path.dirname(resource.path)
else:
workspace_root_path = workspace_root.path

if workspace_package_data and workspace_members:

if workspace_root and workspace_package_data and workspace_members:
# TODO: support glob patterns found in cargo workspaces
for workspace_member_path in workspace_members:
workspace_directory_path = os.path.join(workspace_root_path, workspace_member_path)
workspace_directory_path = os.path.join(workspace_root.path, workspace_member_path)
workspace_directory = codebase.get_resource(path=workspace_directory_path)
if not workspace_directory:
continue
Expand All @@ -103,20 +98,18 @@ def assemble(cls, package_data, resource, codebase, package_adder):
resource.save(codebase)

yield from cls.assemble_from_many_datafiles(
datafile_name_patterns=('Cargo.toml', 'cargo.toml', 'Cargo.lock', 'cargo.lock'),
directory=workspace_directory,
datafile_path_patterns=datafile_path_patterns,
resource=workspace_directory,
codebase=codebase,
package_adder=package_adder,
)
else:
parent_resource = resource.parent(codebase)
if parent_resource:
yield from cls.assemble_from_many_datafiles(
datafile_name_patterns=('Cargo.toml', 'cargo.toml', 'Cargo.lock', 'cargo.lock'),
directory=parent_resource,
codebase=codebase,
package_adder=package_adder,
)
yield from cls.assemble_from_many_datafiles(
datafile_path_patterns=datafile_path_patterns,
resource=workspace_root,
codebase=codebase,
package_adder=package_adder,
)

@classmethod
def update_resource_package_data(cls, workspace, workspace_package_data, resource_package_data, mapping=None):
Expand Down
8 changes: 6 additions & 2 deletions src/packagedcode/chef.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,9 +146,13 @@ def assemble(cls, package_data, resource, codebase, package_adder):
"""
Assemble Package from Chef metadata.rb, then from metadata.json files.
"""
datafile_path_patterns = (
ChefMetadataRbHandler.path_patterns +
ChefMetadataJsonHandler.path_patterns
)
yield from cls.assemble_from_many_datafiles(
datafile_name_patterns=('metadata.rb', 'metadata.json',),
directory=resource.parent(codebase),
datafile_path_patterns=datafile_path_patterns,
resource=resource.parent(codebase),
codebase=codebase,
package_adder=package_adder,
)
Expand Down
31 changes: 14 additions & 17 deletions src/packagedcode/cocoapods.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,18 +151,16 @@ def assemble(cls, package_data, resource, codebase, package_adder):
has_single_podspec = siblings_counts == 1
has_multiple_podspec = siblings_counts > 1

datafile_name_patterns = (
'Podfile.lock',
'Podfile',
datafile_path_patterns = (
PodfileLockHandler.path_patterns +
PodfileHandler.path_patterns
)

if has_single_podspec:
# we can treat all podfile/spec as being for one package
datafile_name_patterns = (sibling_podspecs[0].name,) + datafile_name_patterns

yield from models.DatafileHandler.assemble_from_many_datafiles(
datafile_name_patterns=datafile_name_patterns,
directory=parent,
podspec_path_patterns = (f"*{sibling_podspecs[0].name}",)
yield from cls.assemble_from_many_datafiles(
datafile_path_patterns=podspec_path_patterns + datafile_path_patterns,
resource=parent,
codebase=codebase,
package_adder=package_adder,
)
Expand All @@ -172,11 +170,10 @@ def assemble(cls, package_data, resource, codebase, package_adder):
# as we cannot determine easily which podfile is for which
# podspec
podspec = sibling_podspecs.pop()
datafile_name_patterns = (podspec.name,) + datafile_name_patterns

yield from models.DatafileHandler.assemble_from_many_datafiles(
datafile_name_patterns=datafile_name_patterns,
directory=parent,
podspec_path_patterns = (f"*{podspec.name}",)
yield from cls.assemble_from_many_datafiles(
datafile_path_patterns=podspec_path_patterns + datafile_path_patterns,
resource=parent,
codebase=codebase,
package_adder=package_adder,
)
Expand All @@ -200,9 +197,9 @@ def assemble(cls, package_data, resource, codebase, package_adder):

else:
# has_no_podspec:
yield from models.DatafileHandler.assemble_from_many_datafiles(
datafile_name_patterns=datafile_name_patterns,
directory=parent,
yield from cls.assemble_from_many_datafiles(
datafile_path_patterns=datafile_path_patterns,
resource=parent,
codebase=codebase,
package_adder=package_adder,
)
Expand Down
7 changes: 5 additions & 2 deletions src/packagedcode/golang.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,12 @@ def assemble(cls, package_data, resource, codebase, package_adder):
"""
Always use go.mod first then go.sum
"""
datafile_path_patterns = (
GoModHandler.path_patterns + GoSumHandler.path_patterns
)
yield from cls.assemble_from_many_datafiles(
datafile_name_patterns=('go.mod', 'go.sum',),
directory=resource.parent(codebase),
datafile_path_patterns=datafile_path_patterns,
resource=resource.parent(codebase),
codebase=codebase,
package_adder=package_adder,
)
Expand Down
27 changes: 13 additions & 14 deletions src/packagedcode/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1360,15 +1360,16 @@ def assemble_from_many(
@classmethod
def assemble_from_many_datafiles(
cls,
datafile_name_patterns,
directory,
datafile_path_patterns,
resource,
codebase,
package_adder=add_to_package,
):
"""
Assemble Package and Dependency from package data of the datafiles found
in multiple ``datafile_name_patterns`` name patterns (case- sensitive)
found in the ``directory`` Resource.
in potentially multiple ``datafile_name_patterns`` name patterns
(case- sensitive) found in the ``resource``. A ``resource`` can be a single
file, or a directory in which case we look for manifests in the directory.

Create a Package from the first package data item. Update this package
with other items. Assign to this Package the file tree from the parent
Expand All @@ -1383,22 +1384,20 @@ def assemble_from_many_datafiles(
multiple PackageData for unrelated Packages.
"""
if TRACE:
logger_debug(f'assemble_from_many_datafiles: datafile_name_patterns: {datafile_name_patterns!r}')
logger_debug(f'assemble_from_many_datafiles: datafile_path_patterns: {datafile_path_patterns!r}')

if not codebase.has_single_resource:
siblings = list(directory.children(codebase))
else:
if directory:
siblings = [directory]
else:
siblings = []
siblings = []
if resource and resource.is_file:
siblings = [resource]
elif resource and resource.is_dir:
siblings = list(resource.children(codebase))

pkgdata_resources = []

# we iterate on datafile_name_patterns because their order matters
for datafile_name_pattern in datafile_name_patterns:
for path_pattern in datafile_path_patterns:
for sibling in siblings:
if fnmatchcase(sibling.name, datafile_name_pattern):
if fnmatchcase(sibling.path, path_pattern):
for package_data in sibling.package_data:
package_data = PackageData.from_dict(package_data)
pkgdata_resources.append((package_data, sibling,))
Expand Down
10 changes: 5 additions & 5 deletions src/packagedcode/phpcomposer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ class BasePhpComposerHandler(models.DatafileHandler):

@classmethod
def assemble(cls, package_data, resource, codebase, package_adder):
datafile_name_patterns = (
'composer.json',
'composer.lock',
datafile_path_patterns = (
PhpComposerJsonHandler.path_patterns +
PhpComposerLockHandler.path_patterns
)

if resource.has_parent():
Expand All @@ -37,8 +37,8 @@ def assemble(cls, package_data, resource, codebase, package_adder):
dir_resource = resource

yield from cls.assemble_from_many_datafiles(
datafile_name_patterns=datafile_name_patterns,
directory=dir_resource,
datafile_path_patterns=datafile_path_patterns,
resource=dir_resource,
codebase=codebase,
package_adder=package_adder,
)
Expand Down
10 changes: 6 additions & 4 deletions src/packagedcode/pubspec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,19 @@ class BaseDartPubspecHandler(models.DatafileHandler):

@classmethod
def assemble(cls, package_data, resource, codebase, package_adder):
datafile_name_patterns = \
DartPubspecYamlHandler.path_patterns + DartPubspecLockHandler.path_patterns
datafile_path_patterns = (
DartPubspecYamlHandler.path_patterns +
DartPubspecLockHandler.path_patterns
)

if resource.has_parent():
dir_resource = resource.parent(codebase)
else:
dir_resource = resource

yield from cls.assemble_from_many_datafiles(
datafile_name_patterns=datafile_name_patterns,
directory=dir_resource,
datafile_path_patterns=datafile_path_patterns,
resource=dir_resource,
codebase=codebase,
package_adder=package_adder,
)
Expand Down
26 changes: 13 additions & 13 deletions src/packagedcode/rubygems.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,18 +56,18 @@ def assemble_extracted_gem(cls, package_data, resource, codebase, package_adder)
An assemble implementation shared by handlers for manifests found in an
extracted gem using extractcode.
"""
datafile_name_patterns = (
'metadata.gz-extract/metadata.gz-extract',
'data.gz-extract/*.gemspec',
'data.gz-extract/Gemfile',
'data.gz-extract/Gemfile.lock',
datafile_path_patterns = (
GemMetadataArchiveExtractedHandler.path_patterns +
GemspecHandler.path_patterns +
GemfileHandler.path_patterns +
GemfileLockHandler.path_patterns
)

gemroot = get_ancestor(levels_up=2, resource=resource, codebase=codebase)

yield from cls.assemble_from_many_datafiles(
datafile_name_patterns=datafile_name_patterns,
directory=gemroot,
datafile_path_patterns=datafile_path_patterns,
resource=gemroot,
codebase=codebase,
package_adder=package_adder,
)
Expand Down Expand Up @@ -104,15 +104,15 @@ class BaseGemProjectHandler(models.DatafileHandler):

@classmethod
def assemble(cls, package_data, resource, codebase, package_adder):
datafile_name_patterns = (
'*.gemspec',
'Gemfile',
'Gemfile.lock',
datafile_path_patterns = (
GemspecHandler.path_patterns +
GemfileHandler.path_patterns +
GemfileLockHandler.path_patterns
)

yield from cls.assemble_from_many_datafiles(
datafile_name_patterns=datafile_name_patterns,
directory=resource.parent(codebase),
datafile_path_patterns=datafile_path_patterns,
resource=resource.parent(codebase),
codebase=codebase,
package_adder=package_adder,
)
Expand Down
Loading