diff --git a/docs/source/index.rst b/docs/source/index.rst index 20b837f2..565772fc 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -42,7 +42,7 @@ What can you do PurlDB? :ref:`symbols_and_strings`. - Detect software supply chain issues by mapping package binaries to their corresponding source code - and determining if there are possible discrepancies between sources and binaries (such as with the + and determining if there are possible discrepancies between sources and sources (such as with the XZ utils attack, or sources and binaries, where package may not report the exact source code used to build binaries with the :ref:`deploy_to_devel` mapping analysis. diff --git a/docs/source/purldb/purl_watch.rst b/docs/source/purldb/purl_watch.rst index 12f95697..34f505d3 100644 --- a/docs/source/purldb/purl_watch.rst +++ b/docs/source/purldb/purl_watch.rst @@ -7,7 +7,7 @@ Maintaining an Up-to-Date PurlDB with PurlWatch PurlDB serves as a knowledge base for packages. It is essential to keep this knowledge base updated as new package versions are released daily. PurlWatch is responsible for keeping PurlDB up-to-date. Depending on the PurlDB -size, PurlWatch provides two different approaches. +size PurlWatch provides two different approach. Methods to Keep PurlDB Updated ------------------------------ @@ -48,7 +48,7 @@ watch. It creates watch task for the PURL and enqueues it in RQ for execution. Advantages ~~~~~~~~~~ - Background tasks ensure that the PurlDB remains updated without manual intervention. - - The watch frequency can be customized to balance the resource usage. + - The watch frequency can be customized to balance the resource uses. - Users can define the depth of data collection based on their needs. .. tip:: diff --git a/docs/source/purldb/rest_api.rst b/docs/source/purldb/rest_api.rst index 6a64474c..bd8d1e85 100644 --- a/docs/source/purldb/rest_api.rst +++ b/docs/source/purldb/rest_api.rst @@ -906,9 +906,9 @@ Also each package can have list of ``addon_pipelines`` to run on the package. Find all addon pipelines `here. `_ -If the ``reindex`` flag is set to True, existing package will be rescanned and all the non existing package will be indexed. -If the ``reindex_set`` flag is set to True, then all the package in the same set will be rescanned. - +If ``reindex`` flag is True then existing package will be rescanned, if ``reindex_set`` +is True then all the package in the same set will be rescanned. +If reindex flag is set to true then all the non existing package will be indexed. .. Note:: @@ -970,11 +970,7 @@ Then return a mapping containing: - The number of package urls that are not processable by the index queue. - unsupported_packages - A list of package urls that are not processable by the index queue. - The package indexing queue can handle certain supported purl - types such as npm, pypi, maven, etc. See "supported_ecosystems" - list in - https://github.com/aboutcode-org/purldb/blob/main/packagedb/api.py - for details. + The package indexing queue can only handle npm and maven purls. - unsupported_vers_count - The number of vers range that are not supported by the univers or package_manager. - unsupported_vers @@ -1076,9 +1072,8 @@ Package Update Set List Take a list of purls (where each item is a mapping containing PURL and content_type). -If uuid is given, all purls will be added to the package set if it exists; -otherwise, a new set will be created and all the purls will be added to -that set. +If uuid is given then all purls will be added to package set if it exists else a +new set would be created and all the purls will be added to that new set. .. Note:: @@ -1120,7 +1115,7 @@ Package Set List Return a list of package sets and the package data of packages within -``GET /api/package_sets/`` +``GET /api/projects/0bbdcf88-ad07-4970-9272-7d5f4c82cc7b/`` .. code-block:: json diff --git a/minecode/collectors/maven.py b/minecode/collectors/maven.py index 8f0fb4d3..bf6bd360 100644 --- a/minecode/collectors/maven.py +++ b/minecode/collectors/maven.py @@ -476,9 +476,6 @@ def process_request(purl_str, **kwargs): collect_links = re.compile(r'href="([^"]+)"').findall -collect_links_and_artifact_timestamps = re.compile( - r'\s+(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}|-)' -).findall def check_if_file_name_is_linked_on_page(file_name, links, **kwargs): @@ -675,6 +672,62 @@ def filter_for_artifacts(timestamps_by_links): return timestamps_by_links_filtered +def collect_links_and_artifact_timestamps(text): + # Return a list of sets containing all link locations and their + # corresponding timestamps extracted from a given HTML text. + + # Pattern that matches with https://repo.maven.apache.org/maven2 + maven_apache_pattern = re.compile( + r']*>[^<]*\s+(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}|-)' + ) + maven_apache_matches = maven_apache_pattern.findall(text) + if maven_apache_matches: + return maven_apache_matches + + # Pattern that matces with + # both Apache (UTC) and Nexus (Z) formats + # https://repository.jboss.org/nexus/service/rest/repository/browse/releases/ + # https://repository.jboss.org/nexus/service/rest/repository/browse/public/ + # https://repository.apache.org/snapshots/ + repo_jboss_apache_pattern = re.compile( + r']*>[^<]*\s*\s*((?:[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\s+(?:UTC|Z)\s+\d{4})| )\s*' + ) + repo_jboss_apache_matches = repo_jboss_apache_pattern.findall(text) + # Convert   to empty string for table format + if repo_jboss_apache_matches: + return [ + (item, "" if timestamp == " " else timestamp) + for item, timestamp in repo_jboss_apache_matches + ] + + # Pattern that matches with + # https://repo.spring.io/milestone + repo_spring_pattern = re.compile( + r']*>[^<]*\s+(\d{2}-[A-Z][a-z]{2}-\d{4}\s+\d{2}:\d{2})' + ) + repo_spring_matches = repo_spring_pattern.findall(text) + if repo_spring_matches: + return repo_spring_matches + + # Simple links in
 tags without timestamps (Gradle plugins format)
+    # https://plugins.gradle.org/m2/
+    plugins_gradle_pattern = re.compile(r'
]*>[^<]*
') + plugins_gradle_matches = plugins_gradle_pattern.findall(text) + if plugins_gradle_matches: + # Filter out parent directory link if present + filtered_matches = [] + for href in plugins_gradle_matches: + # Skip parent directory links + if href != "../" and not href.startswith(".."): + filtered_matches.append((href, "")) + + # Only return if we found non-parent links + if filtered_matches: + return filtered_matches + + return [] + + def collect_links_from_text(text, filter): """ Return a mapping of link locations and their timestamps, given HTML `text` @@ -700,7 +753,7 @@ def create_absolute_urls_for_links(text, url, filter): url = url.rstrip("/") timestamps_by_links = collect_links_from_text(text, filter) for link, timestamp in timestamps_by_links.items(): - if not link.startswith(url): + if not link.startswith("http:") and not link.startswith("https:"): link = f"{url}/{link}" timestamps_by_absolute_links[link] = timestamp return timestamps_by_absolute_links @@ -758,23 +811,20 @@ def get_artifact_sha1(artifact_url): return sha1 -def get_classifier_from_artifact_url( - artifact_url, package_version_page_url, package_name, package_version -): +def get_classifier_from_artifact_url(artifact_url, package_name, package_version): """ Return the classifier from a Maven artifact URL `artifact_url`, otherwise return None if a classifier cannot be determined from `artifact_url` """ classifier = None - # https://repo1.maven.org/maven2/net/alchim31/livereload-jvm/0.2.0 - package_version_page_url = package_version_page_url.rstrip("/") - # https://repo1.maven.org/maven2/net/alchim31/livereload-jvm/0.2.0/livereload-jvm-0.2.0 - leading_url_portion = f"{package_version_page_url}/{package_name}-{package_version}" + package_name_version_portion = f"{package_name}-{package_version}" + artifact_url_filename = artifact_url.rsplit("/", 1)[-1] + remaining_url_portion = artifact_url_filename.replace(package_name_version_portion, "") # artifact_url = 'https://repo1.maven.org/maven2/net/alchim31/livereload-jvm/0.2.0/livereload-jvm-0.2.0-onejar.jar' - # ['', '-onejar.jar'] - _, remaining_url_portion = artifact_url.split(leading_url_portion) - # ['-onejar', 'jar'] + # artifact_url_filename = 'livereload-jvm-0.2.0-onejar.jar' + # remaining_url_portion = '-onejar.jar' remaining_url_portions = remaining_url_portion.split(".") + # ['-onejar', 'jar'] if remaining_url_portions and remaining_url_portions[0]: # '-onejar' classifier = remaining_url_portions[0] diff --git a/minecode/management/commands/defederate_packages.py b/minecode/management/commands/defederate_packages.py deleted file mode 100644 index a9668b87..00000000 --- a/minecode/management/commands/defederate_packages.py +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -import sys -import tempfile - -import os -from commoncode.fileutils import walk -from aboutcode.federated import DataFederation -from minecode_pipelines import pipes -from minecode.management.commands import VerboseCommand -from packagedb import models as packagedb_models -from packageurl import PackageURL -import saneyaml - -""" -Utility command to find license oddities. -""" -logger = logging.getLogger(__name__) -logging.basicConfig(stream=sys.stdout) -logger.setLevel(logging.INFO) - -TRACE = False -if TRACE: - logger.setLevel(logging.DEBUG) - - -def yield_purls_from_yaml_files(location): - for root, _, files in walk(location): - for file in files: - if not (file == "purls.yml"): - continue - fp = os.path.join(root, file) - with open(fp) as f: - purl_strs = saneyaml.load(f.read()) or [] - for purl_str in purl_strs: - yield PackageURL.from_string(purl_str) - - -class Command(VerboseCommand): - help = "Find packages with an ambiguous declared license." - - def add_arguments(self, parser): - parser.add_argument("-i", "--input", type=str, help="Define the input file name") - - def handle(self, *args, **options): - logger.setLevel(self.get_verbosity(**options)) - working_path = tempfile.mkdtemp() - - # Clone data and config repo - data_federation = DataFederation.from_url( - name="aboutcode-data", - remote_root_url="https://github.com/aboutcode-data", - ) - data_cluster = data_federation.get_cluster("purls") - - checked_out_repos = {} - for purl_type, data_repository in data_cluster._data_repositories_by_purl_type.items(): - repo_name = data_repository.name - checked_out_repos[repo_name] = pipes.init_local_checkout( - repo_name=repo_name, - working_path=working_path, - logger=logger, - ) - - # iterate through checked out repos and import data - for repo_name, repo_data in checked_out_repos.items(): - repo = repo_data.get("repo") - for purl in yield_purls_from_yaml_files(repo.working_dir): - # TODO: use batch create for efficiency - package = packagedb_models.Package.objects.create( - **purl.to_dict() - ) diff --git a/minecode/management/commands/federate_packages.py b/minecode/management/commands/federate_packages.py deleted file mode 100644 index 5ee6899d..00000000 --- a/minecode/management/commands/federate_packages.py +++ /dev/null @@ -1,100 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -import sys -from aboutcode import hashid - -from aboutcode.federated import DataFederation -from scanpipe.pipes import federatedcode -from minecode_pipelines import pipes -from minecode.management.commands import VerboseCommand -from packagedb import models as packagedb_models - - -""" -Utility command to find license oddities. -""" -logger = logging.getLogger(__name__) -logging.basicConfig(stream=sys.stdout) -logger.setLevel(logging.INFO) - -TRACE = False -if TRACE: - logger.setLevel(logging.DEBUG) - - -PACKAGE_BATCH_SIZE = 1000 - - -def commit_message(commit_batch, total_commit_batch="many"): - from django.conf import settings - - author_name = settings.FEDERATEDCODE_GIT_SERVICE_NAME - author_email = settings.FEDERATEDCODE_GIT_SERVICE_EMAIL - tool_name = "pkg:github/aboutcode-org/purldb" - - return f"""\ - Save PackageURLs from PurlDB ({commit_batch}/{total_commit_batch}) - - Tool: {tool_name}@v{VERSION} - Reference: https://{settings.ALLOWED_HOSTS[0]} - - Signed-off-by: {author_name} <{author_email}> - """ - - -class Command(VerboseCommand): - help = "Find packages with an ambiguous declared license." - - def add_arguments(self, parser): - parser.add_argument("-i", "--input", type=str, help="Define the input file name") - - def handle(self, *args, **options): - logger.setLevel(self.get_verbosity(**options)) - - # Clone data and config repo - data_federation = DataFederation.from_url( - name="aboutcode-data", - remote_root_url="https://github.com/aboutcode-data", - ) - data_cluster = data_federation.get_cluster("purls") - - # TODO: do something more efficient - files_to_commit = [] - commit_batch = 1 - files_per_commit=PACKAGE_BATCH_SIZE - for package in packagedb_models.Package.objects.all(): - package_repo, datafile_path = data_cluster.get_datafile_repo_and_path(purl=package.purl) - purl_file = pipes.write_packageurls_to_file( - repo=package_repo, - relative_datafile_path=datafile_path, - packageurls=[package.purl], - append=True, - ) - if purl_file not in files_to_commit: - files_to_commit.append(purl_file) - - if len(files_to_commit) == files_per_commit: - federatedcode.commit_and_push_changes( - commit_message=commit_message(commit_batch), - repo=package_repo, - files_to_commit=files_to_commit, - logger=logger, - ) - files_to_commit.clear() - commit_batch += 1 - - if files_to_commit: - federatedcode.commit_and_push_changes( - commit_message=commit_message(commit_batch), - repo=package_repo, - files_to_commit=files_to_commit, - logger=logger, - ) diff --git a/minecode/management/commands/import_queue.py b/minecode/management/commands/import_queue.py index ed754e31..d78c9d07 100644 --- a/minecode/management/commands/import_queue.py +++ b/minecode/management/commands/import_queue.py @@ -132,13 +132,14 @@ def process_request(importable_uri): timestamps_by_artifact_links = get_artifact_links(version_page_url) for artifact_link, timestamp in timestamps_by_artifact_links.items(): sha1 = get_artifact_sha1(artifact_link) - classifier = get_classifier_from_artifact_url( - artifact_link, version_page_url, name, version - ) + classifier = get_classifier_from_artifact_url(artifact_link, name, version) qualifiers = None if classifier: qualifiers = f"classifier={classifier}" - release_date = dateutil_parse(timestamp) + if timestamp: + release_date = dateutil_parse(timestamp) + else: + release_date = None package_data = PackageData( type="maven", namespace=namespace, diff --git a/minecode/management/commands/maven_crawler.py b/minecode/management/commands/maven_crawler.py index df6da9cf..baf1fbc7 100644 --- a/minecode/management/commands/maven_crawler.py +++ b/minecode/management/commands/maven_crawler.py @@ -26,5 +26,15 @@ class Command(VerboseCommand): help = "Run a Package request queue." def handle(self, *args, **options): - maven_root_url = "https://repo.maven.apache.org/maven2" - crawl_maven_repo_from_root(root_url=maven_root_url) + # Add the maven root URLs + # Ref: https://github.com/aboutcode-org/purldb/issues/630#issuecomment-3599942716 + maven_root_urls = [ + "https://repo.maven.apache.org/maven2", + "https://repo.spring.io/artifactory/milestone", + "https://plugins.gradle.org/m2", + "https://repository.apache.org/content/groups/snapshots", + "https://repository.jboss.org/nexus/service/rest/repository/browse/releases", + "https://repository.jboss.org/nexus/service/rest/repository/browse/public", + ] + for maven_root_url in maven_root_urls: + crawl_maven_repo_from_root(root_url=maven_root_url) diff --git a/minecode/tests/collectors/test_github.py b/minecode/tests/collectors/test_github.py index 6e3ab2f1..9f077d75 100644 --- a/minecode/tests/collectors/test_github.py +++ b/minecode/tests/collectors/test_github.py @@ -40,13 +40,6 @@ def test_github_get_all_versions(self): "minecode-pipelines/v0.0.1b6", "minecode-pipelines/v0.0.1b7", "minecode-pipelines/v0.0.1b8", - "minecode-pipelines/v0.0.1b9", - "minecode-pipelines/v0.0.1b10", - "minecode-pipelines/v0.0.1b11", - "minecode-pipelines/v0.0.1b12", - "minecode-pipelines/v0.0.1b13", - "minecode-pipelines/v0.0.1b14", - "minecode-pipelines/v0.0.1b15", ] for item in expected: self.assertIn(item, versions) diff --git a/minecode/tests/collectors/test_gitlab.py b/minecode/tests/collectors/test_gitlab.py index 5bfedbd5..edcf1a5a 100644 --- a/minecode/tests/collectors/test_gitlab.py +++ b/minecode/tests/collectors/test_gitlab.py @@ -14,19 +14,79 @@ from minecode.collectors import gitlab from minecode.utils_test import JsonBasedTesting +from unittest import mock + class GitlabPriorityQueueTests(JsonBasedTesting, DjangoTestCase): test_data_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "testfiles") - def test_gitlab_get_all_package_version_author(self): + @mock.patch("requests.get") + def test_gitlab_get_all_package_version_author(self, mock_request_get): repo_path = "xx_network%2Fprimitives" + + mock_data = [ + { + "name": "v0.0.4", + "message": "", + "target": "f7c71b05e13e619feabdc078678d9eb8ff2def3c", + "commit": { + "id": "f7c71b05e13e619feabdc078678d9eb8ff2def3c", + "short_id": "f7c71b05", + "created_at": "2023-12-06T01:50:27.000+00:00", + "parent_ids": ["62d7bebec4063ba4a6da607a78b32bd24b357283"], + "title": "Update copyright", + "message": "Update copyright\n", + "author_name": "Richard T. Carback III", + "author_email": "rick.carback@gmail.com", + "authored_date": "2023-12-06T01:50:27.000+00:00", + "committer_name": "Richard T. Carback III", + "committer_email": "rick.carback@gmail.com", + "committed_date": "2023-12-06T01:50:27.000+00:00", + "trailers": {}, + "extended_trailers": {}, + "web_url": "https://gitlab.com/xx_network/primitives/-/commit/f7c71b05e13e619feabdc078678d9eb8ff2def3c", + }, + "release": None, + "protected": False, + "created_at": None, + }, + { + "name": "v0.0.0", + "message": "", + "target": "a0cd942ed608d4950217aa4099358ab168435e1d", + "commit": { + "id": "a0cd942ed608d4950217aa4099358ab168435e1d", + "short_id": "a0cd942e", + "created_at": "2020-08-05T16:51:29.000+00:00", + "parent_ids": [ + "2aac33c9bb95b47c36f995522801c40c2f4e51a4", + "f99f7a7284da1cd621324ee0a1714a461e055adf", + ], + "title": "Merge branch 'release' into 'master'", + "message": "Merge branch 'release' into 'master'\n\nUse xxn primitives ID and NDF packages\n\nSee merge request xx_network/primitives!1", + "author_name": "Sydney Anne Erickson", + "author_email": "sydney@elixxir.io", + "authored_date": "2020-08-05T16:51:29.000+00:00", + "committer_name": "Sydney Anne Erickson", + "committer_email": "sydney@elixxir.io", + "committed_date": "2020-08-05T16:51:29.000+00:00", + "trailers": {}, + "extended_trailers": {}, + "web_url": "https://gitlab.com/xx_network/primitives/-/commit/a0cd942ed608d4950217aa4099358ab168435e1d", + }, + "release": None, + "protected": False, + "created_at": None, + }, + ] + mock_response = mock.Mock() + mock_response.json.return_value = mock_data + mock_response.raise_for_status.return_value = None + mock_request_get.return_value = mock_response + version_author_list = gitlab.gitlab_get_all_package_version_author(repo_path) expected = [ - ("v0.0.5", "Richard T. Carback III", "rick.carback@gmail.com"), ("v0.0.4", "Richard T. Carback III", "rick.carback@gmail.com"), - ("v0.0.3", "Benjamin Wenger", "ben@privategrity.com"), - ("v0.0.2", "Richard T. Carback III", "rick.carback@gmail.com"), - ("v0.0.1", "Jonathan Wenger", "jono@elixxir.io"), ("v0.0.0", "Sydney Anne Erickson", "sydney@elixxir.io"), ] for item in version_author_list: diff --git a/minecode/tests/collectors/test_maven.py b/minecode/tests/collectors/test_maven.py index 541bf28b..05d31f6c 100644 --- a/minecode/tests/collectors/test_maven.py +++ b/minecode/tests/collectors/test_maven.py @@ -201,7 +201,7 @@ def test_get_merged_ancestor_package_from_maven_package( class MavenCrawlerFunctionsTest(JsonBasedTesting, DjangoTestCase): - test_data_dir = os.path.join(os.path.dirname(__file__), "testfiles") + test_data_dir = os.path.join(os.path.dirname(__file__), "../testfiles") def test_check_if_file_name_is_linked_on_page(self): links = ["foo/", "bar/", "baz/"] @@ -500,12 +500,80 @@ def test_get_artifact_sha1(self, mock_request_get): def test_get_classifier_from_artifact_url(self): artifact_url = "https://repo1.maven.org/maven2/net/alchim31/livereload-jvm/0.2.0/livereload-jvm-0.2.0-onejar.jar" - package_version_page_url = ( - "https://repo1.maven.org/maven2/net/alchim31/livereload-jvm/0.2.0/" - ) package_name = "livereload-jvm" package_version = "0.2.0" classifier = maven.get_classifier_from_artifact_url( - artifact_url, package_version_page_url, package_name, package_version + artifact_url, package_name, package_version ) self.assertEqual("onejar", classifier) + + def test_collect_links_and_artifact_timestamps_repo_maven_apache_org(self): + # https://repo.maven.apache.org/maven2 + with open(self.get_test_loc("maven/html/maven.apache.org/abbot.html")) as file: + text = file.read() + expected = [ + ("1.4.0/", "2015-09-22 16:03"), + ("maven-metadata.xml", "2015-09-24 14:18"), + ] + + self.assertEqual(expected, maven.collect_links_and_artifact_timestamps(text)) + + def test_collect_links_and_artifact_timestamps_repository_jboss_org(self): + # https://repository.jboss.org/nexus/service/rest/repository/browse/public/ + # https://repository.jboss.org/nexus/service/rest/repository/browse/releases/ + with open(self.get_test_loc("maven/html/repository.jboss.org/commons-codec.html")) as file: + text = file.read() + expected = [ + ("1.2/", ""), + ( + "https://repository.jboss.org/nexus/repository/public/apache-codec/commons-codec/maven-metadata.xml", + "Fri Sep 05 09:38:07 Z 2025", + ), + ] + + self.assertEqual(expected, maven.collect_links_and_artifact_timestamps(text)) + + def test_collect_links_and_artifact_timestamps_repository_apache_org(self): + # https://repository.apache.org/snapshots/ + with open(self.get_test_loc("maven/html/repository.apache.org/common-chain.html")) as file: + text = file.read() + expected = [ + ( + "https://repository.apache.org/content/groups/snapshots/commons-chain/commons-chain/1.3-SNAPSHOT/", + "Thu Jul 04 05:45:00 UTC 2013", + ), + ( + "https://repository.apache.org/content/groups/snapshots/commons-chain/commons-chain/2.0-SNAPSHOT/", + "Tue Aug 21 20:26:48 UTC 2018", + ), + ( + "https://repository.apache.org/content/groups/snapshots/commons-chain/commons-chain/maven-metadata.xml.md5", + "Tue Aug 21 20:26:47 UTC 2018", + ), + ( + "https://repository.apache.org/content/groups/snapshots/commons-chain/commons-chain/maven-metadata.xml.sha1", + "Tue Aug 21 20:26:47 UTC 2018", + ), + ] + + self.assertEqual(expected, maven.collect_links_and_artifact_timestamps(text)) + + def test_collect_links_and_artifact_timestamps_repo_spring_io(self): + # https://repo.spring.io/release + with open(self.get_test_loc("maven/html/repo.spring.io/scstest.html")) as file: + text = file.read() + expected = [ + ("0.0.11.M2/", "07-Aug-2019 08:40"), + ("0.0.11.RC2/", "07-Aug-2019 08:36"), + ("maven-metadata.xml", "07-Aug-2019 09:07"), + ] + + self.assertEqual(expected, maven.collect_links_and_artifact_timestamps(text)) + + def test_collect_links_and_artifact_timestamps_plugin_gradle_org(self): + # https://plugins.gradle.org/m2/ + with open(self.get_test_loc("maven/html/plugins.gradle.org/test.html")) as file: + text = file.read() + expected = [("0.0.10/", ""), ("1.0.1/", ""), ("1.1.0/", ""), ("maven-metadata.xml", "")] + + self.assertEqual(expected, maven.collect_links_and_artifact_timestamps(text)) diff --git a/minecode/tests/testfiles/maven/html/maven.apache.org/abbot.html b/minecode/tests/testfiles/maven/html/maven.apache.org/abbot.html new file mode 100644 index 00000000..7b2ce8fd --- /dev/null +++ b/minecode/tests/testfiles/maven/html/maven.apache.org/abbot.html @@ -0,0 +1,29 @@ + + + + + Central Repository: abbot/abbot + + + + + +
+

abbot/abbot

+
+
+
+
+../
+1.4.0/                                            2015-09-22 16:03         -
+maven-metadata.xml                                2015-09-24 14:18       402
+		
+
+
+ + + diff --git a/minecode/tests/testfiles/maven/html/plugins.gradle.org/test.html b/minecode/tests/testfiles/maven/html/plugins.gradle.org/test.html new file mode 100644 index 00000000..b4df3484 --- /dev/null +++ b/minecode/tests/testfiles/maven/html/plugins.gradle.org/test.html @@ -0,0 +1,10 @@ + + + + +
0.0.10/
+
1.0.1/
+
1.1.0/
+
maven-metadata.xml
+ + diff --git a/minecode/tests/testfiles/maven/html/repo.spring.io/scstest.html b/minecode/tests/testfiles/maven/html/repo.spring.io/scstest.html new file mode 100644 index 00000000..b0c8c1f4 --- /dev/null +++ b/minecode/tests/testfiles/maven/html/repo.spring.io/scstest.html @@ -0,0 +1,14 @@ + + + +Index of milestone/com/albertoimpl/test/scstest/releasetest + + +

Index of milestone/com/albertoimpl/test/scstest/releasetest

+
Name                Last modified      Size

+
../
+0.0.11.M2/           07-Aug-2019 08:40    -
+0.0.11.RC2/          07-Aug-2019 08:36    -
+maven-metadata.xml   07-Aug-2019 09:07  449 bytes
+
+
Artifactory Online Server
diff --git a/minecode/tests/testfiles/maven/html/repository.apache.org/common-chain.html b/minecode/tests/testfiles/maven/html/repository.apache.org/common-chain.html new file mode 100644 index 00000000..29937019 --- /dev/null +++ b/minecode/tests/testfiles/maven/html/repository.apache.org/common-chain.html @@ -0,0 +1,59 @@ + + + Index of /groups/snapshots/commons-chain/commons-chain + + + + + + + + +

Index of /groups/snapshots/commons-chain/commons-chain

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameLast ModifiedSizeDescription
Parent Directory
1.3-SNAPSHOT/Thu Jul 04 05:45:00 UTC 2013 +   +
2.0-SNAPSHOT/Tue Aug 21 20:26:48 UTC 2018 +   +
maven-metadata.xml.md5Tue Aug 21 20:26:47 UTC 2018 + 33 +
maven-metadata.xml.sha1Tue Aug 21 20:26:47 UTC 2018 + 41 +
+ + diff --git a/minecode/tests/testfiles/maven/html/repository.jboss.org/commons-codec.html b/minecode/tests/testfiles/maven/html/repository.jboss.org/commons-codec.html new file mode 100644 index 00000000..483c331c --- /dev/null +++ b/minecode/tests/testfiles/maven/html/repository.jboss.org/commons-codec.html @@ -0,0 +1,50 @@ + + + + Index of /apache-codec/commons-codec + + + + + + + + + + +

Index of /apache-codec/commons-codec

+ + + + + + + + + + + + + + + + + + + + + + + + +
NameLast ModifiedSizeDescription
Parent Directory
1.2 +   + +   +
maven-metadata.xml + Fri Sep 05 09:38:07 Z 2025 + + 347 +
+ + diff --git a/minecode_pipelines/__init__.py b/minecode_pipelines/__init__.py index faf2ebbc..7996e45f 100644 --- a/minecode_pipelines/__init__.py +++ b/minecode_pipelines/__init__.py @@ -8,4 +8,4 @@ # -VERSION = "0.0.1b59" +VERSION = "0.0.1b25" diff --git a/minecode_pipelines/miners/cargo.py b/minecode_pipelines/miners/cargo.py new file mode 100644 index 00000000..2760aff0 --- /dev/null +++ b/minecode_pipelines/miners/cargo.py @@ -0,0 +1,89 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from minecode_pipelines.pipes.cargo import store_cargo_packages +from scanpipe.pipes.federatedcode import commit_and_push_changes +import json +from pathlib import Path +from django.conf import settings +from scancodeio import VERSION +from aboutcode.pipeline import LoopProgress + + +def cargo_commit_message(commit_batch, total_commit_batch="many"): + author_name = settings.FEDERATEDCODE_GIT_SERVICE_NAME + author_email = settings.FEDERATEDCODE_GIT_SERVICE_EMAIL + tool_name = "pkg:github/aboutcode-org/scancode.io" + + return f"""\ + Collect PackageURLs from crates.io index ({commit_batch}/{total_commit_batch}) + + Tool: {tool_name}@v{VERSION} + Reference: https://{settings.ALLOWED_HOSTS[0]} + + Signed-off-by: {author_name} <{author_email}> + """ + + +def process_cargo_packages(cargo_index_repo, cloned_data_repo, logger): + """Mine and publish Cargo PackageURLs from Crates.io package index.""" + + base_path = Path(cargo_index_repo.working_tree_dir) + batch_size = 4000 + file_counter = 0 + purl_files = [] + commit_count = 1 + + package_dir = [p for p in base_path.iterdir() if p.is_dir() and not p.name.startswith(".")] + package_paths = [f for dir in package_dir for f in dir.rglob("*") if f.is_file()] + package_count = len(package_paths) + + progress = LoopProgress( + total_iterations=package_count, + logger=logger, + ) + + logger(f"Mine PackageURL for {package_count:,d} Cargo packages.") + for path in progress.iter(package_paths): + packages = [] + + with open(path, encoding="utf-8") as f: + for line_number, line in enumerate(f, start=1): + line = line.strip() + if not line: + continue + try: + packages.append(json.loads(line)) + except json.JSONDecodeError as e: + logger(f"Skipping invalid JSON in {path} at line {line_number}: {e}") + + file_counter += 1 + result = store_cargo_packages(packages, cloned_data_repo) + if result: + purl_file, _ = result + purl_files.append(purl_file) + + if file_counter % batch_size == 0 and purl_files: + if commit_and_push_changes( + repo=cloned_data_repo, + files_to_commit=purl_files, + commit_message=cargo_commit_message(commit_count), + logger=logger, + ): + commit_count += 1 + purl_files.clear() + + commit_and_push_changes( + repo=cloned_data_repo, + files_to_commit=purl_files, + commit_message=cargo_commit_message(commit_count, commit_count), + logger=logger, + ) + logger(f"Processed PackageURL for {file_counter:,d} Cargo packages.") + logger(f"Pushed new PackageURL in {commit_count:,d} commits.") diff --git a/minecode_pipelines/miners/composer.py b/minecode_pipelines/miners/composer.py new file mode 100644 index 00000000..a58adf8d --- /dev/null +++ b/minecode_pipelines/miners/composer.py @@ -0,0 +1,103 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json +from minecode_pipelines.utils import get_temp_file +import requests +from packageurl import PackageURL + + +def get_composer_packages(): + """ + Fetch all Composer packages from Packagist and save them to a temporary JSON file. + Response example: + { + "packageNames" ["0.0.0/composer-include-files", "0.0.0/laravel-env-shim"] + } + """ + + response = requests.get("https://packagist.org/packages/list.json") + if not response.ok: + return + + packages = response.json() + temp_file = get_temp_file("ComposerPackages", "json") + with open(temp_file, "w", encoding="utf-8") as f: + json.dump(packages, f, indent=4) + + return temp_file + + +def get_composer_purl(vendor, package): + """ + Fetch all available Package URLs (purls) for a Composer package from Packagist. + Response example: + { + "minified": "composer/2.0", + "packages": [ + { + "monolog/monolog": { + "0": { + "name": "monolog/monolog", + "version": "3.9.0" + } + } + } + ], + "security-advisories": [ + { + "advisoryId": "PKSA-dmw8-jd8k-q3c6", + "affectedVersions": ">=1.8.0,<1.12.0" + } + ] + } + get_composer_purl("monolog", "monolog") + -> ["pkg:composer/monolog/monolog@3.9.0", "pkg:composer/monolog/monolog@3.8.0", ...] + """ + purls = [] + url = f"https://repo.packagist.org/p2/{vendor}/{package}.json" + + try: + response = requests.get(url, timeout=10) + response.raise_for_status() + except requests.RequestException: + return purls + + data = response.json() + packages = data.get("packages", {}) + releases = packages.get(f"{vendor}/{package}", []) + + for release in releases: + version = release.get("version") + if version: + purl = PackageURL( + type="composer", + namespace=vendor, + name=package, + version=version, + ) + purls.append(purl.to_string()) + + return purls + + +def load_composer_packages(packages_file): + """Load and return a list of (vendor, package) tuples from a JSON file.""" + with open(packages_file, encoding="utf-8") as f: + packages_data = json.load(f) + + package_names = packages_data.get("packageNames", []) + result = [] + + for item in package_names: + if "/" in item: + vendor, package = item.split("/", 1) + result.append((vendor, package)) + + return result diff --git a/minecode_pipelines/miners/conan.py b/minecode_pipelines/miners/conan.py new file mode 100644 index 00000000..1c7a13a4 --- /dev/null +++ b/minecode_pipelines/miners/conan.py @@ -0,0 +1,68 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path +import saneyaml + +from scanpipe.pipes.federatedcode import commit_changes +from scanpipe.pipes.federatedcode import push_changes +from minecode_pipelines import VERSION +from minecode_pipelines.pipes.conan import store_conan_packages + +PACKAGE_BATCH_SIZE = 1000 + + +def mine_and_publish_conan_packageurls(conan_index_repo, cloned_data_repo, logger): + base_path = Path(conan_index_repo.working_dir) + + yml_files = [] + for file_path in base_path.glob("recipes/**/*"): + if not file_path.name == "config.yml": + continue + yml_files.append(file_path) + + file_counter = 0 + purl_files = [] + purls = [] + + total_files = len(yml_files) + logger(f"Processing total files: {total_files}") + for idx, file_path in enumerate(yml_files, start=1): + # Example: file_path = Path("repo_path/recipes/7zip/config.yml") + # - file_path.parts = ("repo_path", "recipes", "7zip", "config.yml") + # - file_path.parts[-2] = "7zip" (the package name) + package = file_path.parts[-2] + with open(file_path, encoding="utf-8") as f: + versions = saneyaml.load(f) + + if not versions: + continue + + file_counter += 1 + push_commit = file_counter >= PACKAGE_BATCH_SIZE or idx == total_files + + result_store = store_conan_packages(package, versions, cloned_data_repo) + if result_store: + purl_file, base_purl = result_store + logger(f"writing packageURLs for package: {base_purl} at: {purl_file}") + + purl_files.append(purl_file) + purls.append(str(base_purl)) + + if push_commit: + commit_changes( + repo=cloned_data_repo, + files_to_commit=purl_files, + purls=purls, + mine_type="packageURL", + tool_name="pkg:pypi/minecode-pipelines", + tool_version=VERSION, + ) + push_changes(repo=cloned_data_repo) + file_counter = 0 diff --git a/minecode_pipelines/miners/cpan.py b/minecode_pipelines/miners/cpan.py deleted file mode 100644 index af173c33..00000000 --- a/minecode_pipelines/miners/cpan.py +++ /dev/null @@ -1,159 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import gzip -import requests - -from bs4 import BeautifulSoup -from packageurl import PackageURL - -from minecode_pipelines.utils import get_temp_file - -""" -Visitors for cpan and cpan-like perl package repositories. -""" - - -CPAN_REPO = "https://www.cpan.org/" -CPAN_TYPE = "cpan" - - -def get_cpan_packages(cpan_repo=CPAN_REPO, logger=None): - """ - Get cpan package names parsed from the `02packages.details.txt` - which conatins a list of all modules and their respective - package archive paths. We parse the package names and their respective - path_prefixes with author page path from this list. - """ - cpan_packages_url = cpan_repo + "modules/02packages.details.txt.gz" - packages_archive = get_temp_file(file_name="cpan_packages", extension=".gz") - packages_content = get_temp_file(file_name="cpan_packages", extension=".txt") - response = requests.get(cpan_packages_url, stream=True) - with open(packages_archive, 'wb') as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) - - with gzip.open(packages_archive, "rb") as f_in: - with open(packages_content, "wb") as f_out: - f_out.writelines(f_in) - - with open(packages_content, 'r', encoding='utf-8') as file: - packages_content = file.read() - - package_path_by_name = {} - - # The ``modules/02packages.details.txt`` file has the following section - # at the beginning of the file: - # - # File: 02packages.details.txt - # URL: http://www.cpan.org/modules/02packages.details.txt - # Description: Package names found in directory $CPAN/authors/id/ - # Columns: package name, version, path - # Intended-For: Automated fetch routines, namespace documentation. - # Written-By: PAUSE version 1.005 - # Line-Count: 268940 - # Last-Updated: Mon, 29 Sep 2025 22:29:02 GMT - # - # This information is there in first 10 lines, and the last line is an - # empty line, both of which we are ignoring below - - modules = packages_content.split("\n")[9:-1] - - # A sample line from this module list looks like this: - # - # Crypt::Passphrase::SHA1::Base64 0.021 L/LE/LEONT/Crypt-Passphrase-0.021.tar.gz - - for module in modules: - info = [section for section in module.split(" ") if section] - - # This is like: L/LE/LEONT/Crypt-Passphrase-0.021.tar.gz - package_path = info[-1] - path_segments = package_path.split("/") - filename = path_segments.pop() - path_prefix = "/".join(path_segments) - - name_version = filename.replace(".tar.gz", "").split("-") - _version = name_version.pop() - name = "-".join(name_version) - - # for the above example: name: Crypt-Passphrase, path_prefix: L/LE/LEONT/ - package_path_by_name[name] = path_prefix - - return package_path_by_name - - -def get_cpan_packageurls(name, path_prefix, logger=None): - """ - Given a package name and it's path_prefix (author page path) - return a list of packageURLs for that package. - - An author page (like https://www.cpan.org/authors/id/P/PT/PTC/) lists - all versions of all packages released by the author, so we can scrape - all the packageURLs from this author packages index. - """ - - author_name = path_prefix.split("/")[-1] - - packageurls = [] - - # file extensions found in cpan index - ignorable_extensions = [".meta", ".readme", ".tar.gz"] - - cpan_authors_path = "/authors/id/" - cpan_authors_url = CPAN_REPO + cpan_authors_path - - cpan_author_page_url = cpan_authors_url + path_prefix - - response = requests.get(cpan_author_page_url) - if not response.ok: - return packageurls - - if logger: - logger(f"Getting package versions for {name} from {cpan_author_page_url}") - - soup = BeautifulSoup(response.text, "html.parser") - - # We get all the listed packages in the author page index - package_list = soup.find("ul") - if not package_list: - return packageurls - - package_list_elements = package_list.text.split("\n") - - package_elements = [ - element.replace(" ", "") - for element in package_list_elements - if element and element not in {" Parent Directory", " CHECKSUMS"} - ] - - versions = [] - for package_file in package_elements: - for extension in ignorable_extensions: - if extension in package_file: - package_file = package_file.replace(extension, "") - - name_version = package_file.split("-") - version = name_version.pop() - package_name = "-".join(name_version) - if package_name != name: - continue - - versions.append(version) - - unique_versions = list(set(versions)) - for version in unique_versions: - purl = PackageURL( - type=CPAN_TYPE, - namespace=author_name, - name=name, - version=version, - ) - packageurls.append(purl.to_string()) - - return packageurls diff --git a/minecode_pipelines/miners/cran.py b/minecode_pipelines/miners/cran.py new file mode 100644 index 00000000..09bac091 --- /dev/null +++ b/minecode_pipelines/miners/cran.py @@ -0,0 +1,66 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import json +from pathlib import Path +import requests +from packageurl import PackageURL + + +def fetch_cran_db(output_file="cran_db.json") -> Path: + """ + Download the CRAN package database (~250MB JSON) in a memory-efficient way. + Saves it to a file instead of loading everything into memory. + """ + + url = "https://crandb.r-pkg.org/-/all" + output_path = Path(output_file) + + with requests.get(url, stream=True) as response: + response.raise_for_status() + with output_path.open("wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + + return output_path + + +def extract_cran_packages(json_file_path: str) -> list: + """ + Extract package names and their versions from a CRAN DB JSON file. + ex: + { + "AATtools": { + "_id": "AATtools", + "_rev": "8-9ebb721d05b946f2b437b49e892c9e8c", + "name": "AATtools", + "versions": { + "0.0.1": {...}, + "0.0.2": {...}, + "0.0.3": {...} + } + } + """ + db_path = Path(json_file_path) + if not db_path.exists(): + raise FileNotFoundError(f"File not found: {db_path}") + + with open(db_path, encoding="utf-8") as f: + data = json.load(f) + + for pkg_name, pkg_data in data.items(): + versions = list(pkg_data.get("versions", {}).keys()) + purls = [] + for version in versions: + purl = PackageURL( + type="cran", + name=pkg_name, + version=version, + ) + purls.append(purl.to_string()) + yield purls diff --git a/minecode_pipelines/miners/swift.py b/minecode_pipelines/miners/swift.py new file mode 100644 index 00000000..e146f765 --- /dev/null +++ b/minecode_pipelines/miners/swift.py @@ -0,0 +1,99 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import shutil +import subprocess +from urllib.parse import urlparse + +""" +Clone the Swift Index repo (https://github.com/SwiftPackageIndex/PackageList) and the Minecode Pipelines Swift repo. +Read the packages.json file from the Swift Index repo to get a list of Git repositories. +Fetch the tags for each repo using the git ls-remote command, +then create package URLs for each repo with its version and store them in the Minecode Pipelines Swift repo. +""" + + +def is_safe_repo_url(repo_url: str) -> bool: + """Return True if the URL is HTTPS GitHub with .git suffix or has at least two path segments.""" + parsed = urlparse(repo_url) + return ( + parsed.scheme == "https" + and parsed.netloc == "github.com" + and parsed.path.endswith(".git") + or parsed.path.count("/") >= 2 + ) + + +def fetch_git_tags_raw(repo_url: str, timeout: int = 60, logger=None) -> str | None: + """Run `git ls-remote` on a GitHub repo and return raw output, or None on error.""" + git_executable = shutil.which("git") + if git_executable is None: + logger("Git executable not found in PATH") + return None + + if not is_safe_repo_url(repo_url): + raise ValueError(f"Unsafe repo URL: {repo_url}") + + try: + result = subprocess.run( # NOQA + [git_executable, "ls-remote", repo_url], + capture_output=True, + text=True, + check=True, + timeout=timeout, + ) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + logger(f"Failed to fetch tags for {repo_url}: {e}") + except subprocess.TimeoutExpired: + logger(f"Timeout fetching tags for {repo_url}") + return None + + +# FIXME duplicated with miners github +def split_org_repo(url_like): + """ + Given a URL-like string to a GitHub repo or a repo name as in org/name, + split and return the org and name. + + For example: + >>> split_org_repo('foo/bar') + ('foo', 'bar') + >>> split_org_repo('https://api.github.com/repos/foo/bar/') + ('foo', 'bar') + >>> split_org_repo('github.com/foo/bar/') + ('foo', 'bar') + >>> split_org_repo('git://github.com/foo/bar.git') + ('foo', 'bar') + """ + segments = [s.strip() for s in url_like.split("/") if s.strip()] + if not len(segments) >= 2: + raise ValueError(f"Not a GitHub-like URL: {url_like}") + org = segments[-2] + name = segments[-1] + if name.endswith(".git"): + name, _, _ = name.rpartition(".git") + return org, name + + +def get_tags_and_commits_from_git_output(git_ls_remote): + """ + Yield tuples of (tag, commit), given a git ls-remote output + """ + tags_and_commits = [] + for line in git_ls_remote.split("\n"): + # line: kjwfgeklngelkfjofjeo123 refs/tags/1.2.3 + line_segments = line.split("\t") + # segments: ["kjwfgeklngelkfjofjeo123", "refs/tags/1.2.3"] + if len(line_segments) > 1 and ( + line_segments[1].startswith("refs/tags/") or line_segments[1] == "HEAD" + ): + commit = line_segments[0] + tag = line_segments[1].replace("refs/tags/", "") + tags_and_commits.append((tag, commit)) + return tags_and_commits diff --git a/minecode_pipelines/pipelines/__init__.py b/minecode_pipelines/pipelines/__init__.py index 0dee9751..0ebd311c 100644 --- a/minecode_pipelines/pipelines/__init__.py +++ b/minecode_pipelines/pipelines/__init__.py @@ -8,22 +8,17 @@ # -import logging import shutil -import time -from collections.abc import Callable from collections.abc import Iterable -from pathlib import Path +import logging +from minecode_pipelines import pipes +from minecode_pipelines.pipes import write_packageurls_to_file -from aboutcode.federated import DataCluster from aboutcode.federated import DataFederation from aboutcode.pipeline import LoopProgress from scanpipe.pipelines import Pipeline from scanpipe.pipes import federatedcode -from minecode_pipelines import pipes -from minecode_pipelines.pipes import write_packageurls_to_file - module_logger = logging.getLogger(__name__) @@ -38,18 +33,12 @@ class MineCodeBasePipeline(Pipeline): download_inputs = False - # Control wether to ovewrite or append mined purls to existing `purls.yml` file - append_purls = False - - checked_out_repos = {} - @classmethod def steps(cls): return ( cls.check_federatedcode_eligibility, cls.create_federatedcode_working_dir, # Add step(s) for downloading/cloning resource as required. - cls.fetch_federation_config, cls.mine_and_publish_packageurls, cls.delete_working_dir, ) @@ -68,7 +57,6 @@ def packages_count(self) -> int: Return the estimated number of packages for which PackageURLs are to be mined. Used by ``mine_and_publish_packageurls`` to log the progress of PackageURL mining. - Note: If estimating package count is not feasable return `None` """ raise NotImplementedError @@ -83,26 +71,68 @@ def create_federatedcode_working_dir(self): """Create temporary working dir.""" self.working_path = federatedcode.create_federatedcode_working_dir() - def fetch_federation_config(self): - """Fetch config for PackageURL Federation.""" + def mine_and_publish_packageurls(self): + """Mine and publish PackageURLs.""" data_federation = DataFederation.from_url( name="aboutcode-data", remote_root_url="https://github.com/aboutcode-data", ) - self.data_cluster = data_federation.get_cluster("purls") + data_cluster = data_federation.get_cluster("purls") + checked_out_repos = {} + batch_size = 4000 + total_file_processed_count = 0 + total_commit_count = 0 + package_count = self.packages_count() + progress = LoopProgress( + total_iterations=package_count, + logger=self.log, + progress_step=1, + ) - def mine_and_publish_packageurls(self): - """Mine and publish PackageURLs.""" + self.log(f"Mine PackageURL for {package_count:,d} packages.") + for base, purls in progress.iter(self.mine_packageurls()): + package_repo, datafile_path = data_cluster.get_datafile_repo_and_path(purl=base) - _mine_and_publish_packageurls( - packageurls=self.mine_packageurls(), - total_package_count=self.packages_count(), - data_cluster=self.data_cluster, - checked_out_repos=self.checked_out_repos, - working_path=self.working_path, - append_purls=self.append_purls, - commit_msg_func=self.commit_message, - logger=self.log, + if package_repo not in checked_out_repos: + checked_out_repos[package_repo] = pipes.init_local_checkout( + repo_name=package_repo, + working_path=self.working_path, + logger=self.log, + ) + + checkout = checked_out_repos[package_repo] + + purl_file = write_packageurls_to_file( + repo=checkout["repo"], + relative_datafile_path=datafile_path, + packageurls=sorted(purls), + ) + checkout["file_to_commit"].append(purl_file) + checkout["file_processed_count"] += 1 + + if len(checkout["file_to_commit"]) > batch_size: + pipes.commit_and_push_checkout( + local_checkout=checkout, + commit_message=self.commit_message(checkout["commit_count"] + 1), + logger=self.log, + ) + + for checkout in checked_out_repos.values(): + final_commit_count = checkout["commit_count"] + 1 + pipes.commit_and_push_checkout( + local_checkout=checkout, + commit_message=self.commit_message( + commit_count=final_commit_count, + total_commit_count=final_commit_count, + ), + logger=self.log, + ) + total_commit_count += checkout["commit_count"] + total_file_processed_count += checkout["file_processed_count"] + + self.log(f"Processed PackageURL for {total_file_processed_count:,d} NuGet packages.") + self.log( + f"Pushed new PackageURL in {total_commit_count:,d} commits in {len(checked_out_repos):,d} repos." ) def delete_working_dir(self): @@ -139,90 +169,3 @@ def log(self, message, level=logging.INFO): print(message) message = message.replace("\r", "\\r").replace("\n", "\\n") self.append_to_log(message) - - -def _mine_and_publish_packageurls( - packageurls: Iterable, - total_package_count: int, - data_cluster: DataCluster, - checked_out_repos: dict, - working_path: Path, - append_purls: bool, - commit_msg_func: Callable, - logger: Callable, - batch_size: int = 4000, - checkpoint_func: Callable = None, - checkpoint_freq: int = 30, -): - """Mine and publish PackageURLs.""" - total_file_processed_count = 0 - total_commit_count = 0 - iterator = packageurls - - last_checkpoint_call = time.time() - checkpoint_interval = checkpoint_freq * 60 - - if total_package_count: - progress = LoopProgress( - total_iterations=total_package_count, - logger=logger, - progress_step=1, - ) - iterator = progress.iter(iterator) - logger(f"Mine PackageURL for {total_package_count:,d} packages.") - - for base, purls in iterator: - if not purls or not base: - continue - - package_repo, datafile_path = data_cluster.get_datafile_repo_and_path(purl=base) - if package_repo not in checked_out_repos: - checked_out_repos[package_repo] = pipes.init_local_checkout( - repo_name=package_repo, - working_path=working_path, - logger=logger, - ) - - checkout = checked_out_repos[package_repo] - purl_file = write_packageurls_to_file( - repo=checkout["repo"], - relative_datafile_path=datafile_path, - packageurls=purls, - append=append_purls, - ) - checkout["file_to_commit"].add(purl_file) - checkout["file_processed_count"] += 1 - - if len(checkout["file_to_commit"]) > batch_size: - pipes.commit_and_push_checkout( - local_checkout=checkout, - commit_message=commit_msg_func(checkout["commit_count"] + 1), - logger=logger, - ) - - time_now = time.time() - checkpoint_due = time_now - last_checkpoint_call >= checkpoint_interval - if checkpoint_func and checkpoint_due: - checkpoint_func() - last_checkpoint_call = time_now - - for checkout in checked_out_repos.values(): - final_commit_count = checkout["commit_count"] + 1 - pipes.commit_and_push_checkout( - local_checkout=checkout, - commit_message=commit_msg_func( - commit_count=final_commit_count, - total_commit_count=final_commit_count, - ), - logger=logger, - ) - total_commit_count += checkout["commit_count"] - total_file_processed_count += checkout["file_processed_count"] - - if checkpoint_func: - checkpoint_func() - - logger(f"Processed PackageURL for {total_file_processed_count:,d} packages.") - logger( - f"Pushed new PackageURL in {total_commit_count:,d} commits in {len(checked_out_repos):,d} repos." - ) diff --git a/minecode_pipelines/pipelines/mine_alpine.py b/minecode_pipelines/pipelines/mine_alpine.py index 63b447c2..38323ac2 100644 --- a/minecode_pipelines/pipelines/mine_alpine.py +++ b/minecode_pipelines/pipelines/mine_alpine.py @@ -20,29 +20,36 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.pipelines import MineCodeBasePipeline +from scanpipe.pipelines import Pipeline +from scanpipe.pipes import federatedcode +from minecode_pipelines import pipes from minecode_pipelines.pipes import alpine -class MineAlpine(MineCodeBasePipeline): - """Mine PackageURLs from alpine index and publish them to FederatedCode.""" +class MineAlpine(Pipeline): + """ + Mine all packageURLs from an alpine index and publish them to + a FederatedCode repo. + """ @classmethod def steps(cls): return ( cls.check_federatedcode_eligibility, - cls.create_federatedcode_working_dir, - cls.fetch_federation_config, - cls.mine_and_publish_alpine_packageurls, - cls.delete_working_dir, + cls.collect_packages_from_alpine, + cls.delete_cloned_repos, ) - def mine_and_publish_alpine_packageurls(self): - alpine.mine_and_publish_alpine_packageurls( - data_cluster=self.data_cluster, - checked_out_repos=self.checked_out_repos, - working_path=self.working_path, - commit_msg_func=self.commit_message, - logger=self.log, - ) + def check_federatedcode_eligibility(self): + """ + Check if the project fulfills the following criteria for + pushing the project result to FederatedCode. + """ + federatedcode.check_federatedcode_configured_and_available(logger=self.log) + + def collect_packages_from_alpine(self): + self.repos = alpine.collect_packages_from_alpine(logger=self.log) + + def delete_cloned_repos(self): + pipes.delete_cloned_repos(repos=self.repos, logger=self.log) diff --git a/minecode_pipelines/pipelines/mine_cargo.py b/minecode_pipelines/pipelines/mine_cargo.py index fa556974..eed7b793 100644 --- a/minecode_pipelines/pipelines/mine_cargo.py +++ b/minecode_pipelines/pipelines/mine_cargo.py @@ -20,45 +20,53 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from pathlib import Path - -from minecode_pipelines.pipes import cargo -from minecode_pipelines.pipelines import MineCodeBasePipeline +import os +from scanpipe.pipelines import Pipeline from scanpipe.pipes import federatedcode +from minecode_pipelines.miners import cargo +from minecode_pipelines import pipes +MINECODE_DATA_CARGO_REPO = os.environ.get( + "MINECODE_DATA_CARGO_REPO", "https://github.com/aboutcode-data/minecode-data-cargo-test" +) +MINECODE_CARGO_INDEX_REPO = "https://github.com/rust-lang/crates.io-index" -class MineCargo(MineCodeBasePipeline): - """Pipeline to mine Cargo (crates.io) packages and publish them to FederatedCode.""" - MINECODE_CARGO_INDEX_REPO = "https://github.com/rust-lang/crates.io-index" +class MineCargo(Pipeline): + """Pipeline to mine Cargo (crates.io) packages and publish them to FederatedCode.""" @classmethod def steps(cls): return ( cls.check_federatedcode_eligibility, - cls.create_federatedcode_working_dir, - cls.clone_cargo_index, - cls.fetch_federation_config, - cls.mine_and_publish_packageurls, - cls.delete_working_dir, + cls.clone_cargo_repos, + cls.mine_and_publish_cargo_packageurls, + cls.delete_cloned_repos, ) - def clone_cargo_index(self): - """Clone the Cargo index Repo.""" - self.cargo_index_repo = federatedcode.clone_repository( - repo_url=self.MINECODE_CARGO_INDEX_REPO, - clone_path=self.working_path / "crates.io-index", - logger=self.log, - ) + def check_federatedcode_eligibility(self): + """ + Check if the project fulfills the following criteria for + pushing the project result to FederatedCode. + """ + federatedcode.check_federatedcode_configured_and_available(logger=self.log) + + def clone_cargo_repos(self): + """ + Clone the Cargo-related repositories (index, data, and pipelines config) + and store their Repo objects in the corresponding instance variables. + """ + self.cargo_index_repo = federatedcode.clone_repository(MINECODE_CARGO_INDEX_REPO) + self.cloned_data_repo = federatedcode.clone_repository(MINECODE_DATA_CARGO_REPO) + + self.log(f"{MINECODE_CARGO_INDEX_REPO} repo cloned at: {self.cargo_index_repo.working_dir}") + self.log(f"{MINECODE_DATA_CARGO_REPO} repo cloned at: {self.cloned_data_repo.working_dir}") - def packages_count(self): - base_path = Path(self.cargo_index_repo.working_tree_dir) - package_dir = [p for p in base_path.iterdir() if p.is_dir() and not p.name.startswith(".")] - return sum(1 for dir in package_dir for f in dir.rglob("*") if f.is_file()) + def mine_and_publish_cargo_packageurls(self): + cargo.process_cargo_packages(self.cargo_index_repo, self.cloned_data_repo, self.log) - def mine_packageurls(self): - """Yield PackageURLs from Cargo index.""" - return cargo.mine_cargo_packageurls( - cargo_index_repo=self.cargo_index_repo, + def delete_cloned_repos(self): + pipes.delete_cloned_repos( + repos=[self.cargo_index_repo, self.cloned_data_repo], logger=self.log, ) diff --git a/minecode_pipelines/pipelines/mine_composer.py b/minecode_pipelines/pipelines/mine_composer.py index 8e734d3d..a03bf811 100644 --- a/minecode_pipelines/pipelines/mine_composer.py +++ b/minecode_pipelines/pipelines/mine_composer.py @@ -20,85 +20,63 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from datetime import datetime +import os +from scanpipe.pipelines import Pipeline from scanpipe.pipes import federatedcode from minecode_pipelines import pipes -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipes import composer -from minecode_pipelines.pipes.composer import mine_composer_packages, PACKAGE_BATCH_SIZE -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO +from minecode_pipelines.pipes.composer import mine_composer_packages +from minecode_pipelines.pipes.composer import mine_and_publish_composer_purls +MINECODE_COMPOSER_GIT_URL = os.environ.get( + "MINECODE_COMPOSER_GIT_URL", "https://github.com/aboutcode-data/minecode-data-composer-test" +) -class MineComposer(MineCodeBasePipeline): + +class MineComposer(Pipeline): """ - Pipeline to mine Composer PHP packages and publish them to FederatedCode. + Mine all packageURLs from a composer index and publish them to a FederatedCode repo. """ - pipeline_config_repo = "https://github.com/aboutcode-data/minecode-pipelines-config/" - checkpoint_path = "composer/checkpoints.json" - checkpoint_freq = 200 - @classmethod def steps(cls): return ( cls.check_federatedcode_eligibility, - cls.create_federatedcode_working_dir, - cls.fetch_checkpoint_and_start_index, - cls.fetch_federation_config, - cls.mine_and_publish_packageurls, - cls.delete_working_dir, - ) - - def fetch_checkpoint_and_start_index(self): - self.checkpoint_config_repo = federatedcode.clone_repository( - repo_url=self.pipeline_config_repo, - clone_path=self.working_path / "minecode-pipelines-config", - logger=self.log, - ) - checkpoint = pipes.get_checkpoint_from_file( - cloned_repo=self.checkpoint_config_repo, - path=self.checkpoint_path, + cls.clone_composer_repo, + cls.mine_and_publish_composer_purls, ) - self.start_index = checkpoint.get("start_index", 0) - self.log(f"start_index: {self.start_index}") + def check_federatedcode_eligibility(self): + """ + Check if the project fulfills the following criteria for + pushing the project result to FederatedCode. + """ + federatedcode.check_federatedcode_configured_and_available(logger=self.log) - def packages_count(self): - return len(self.composer_packages) if self.composer_packages else None + def clone_composer_repo(self): + """ + Clone the federatedcode composer url and return the Repo object + """ + self.cloned_data_repo = federatedcode.clone_repository(MINECODE_COMPOSER_GIT_URL) + self.cloned_config_repo = federatedcode.clone_repository(MINECODE_PIPELINES_CONFIG_REPO) - def mine_packageurls(self): - self.composer_packages = mine_composer_packages() - return composer.mine_composer_packageurls( - packages=self.composer_packages, - start_index=self.start_index, - ) + def mine_and_publish_composer_purls(self): + """ + Mine Composer package names from Composer indexes and generate + package URLs (pURLs) for all mined Composer packages. + """ - def mine_and_publish_packageurls(self): - """Mine and publish PackageURLs.""" - _mine_and_publish_packageurls( - packageurls=self.mine_packageurls(), - total_package_count=self.packages_count(), - data_cluster=self.data_cluster, - checked_out_repos=self.checked_out_repos, - working_path=self.working_path, - append_purls=self.append_purls, - commit_msg_func=self.commit_message, + composer_packages = mine_composer_packages() + mine_and_publish_composer_purls( + packages=composer_packages, + cloned_data_repo=self.cloned_data_repo, + cloned_config_repo=self.cloned_config_repo, logger=self.log, - checkpoint_func=self.save_check_point, - checkpoint_freq=self.checkpoint_freq, ) - def save_check_point(self): - checkpoint = { - "date": str(datetime.now()), - "start_index": self.start_index + self.checkpoint_freq * PACKAGE_BATCH_SIZE, - } - - self.log(f"Saving checkpoint: {checkpoint}") - pipes.update_checkpoints_in_github( - checkpoint=checkpoint, - cloned_repo=self.checkpoint_config_repo, - path=self.checkpoint_path, + def delete_cloned_repos(self): + pipes.delete_cloned_repos( + repos=[self.cloned_data_repo, self.cloned_config_repo], logger=self.log, ) diff --git a/minecode_pipelines/pipelines/mine_conan.py b/minecode_pipelines/pipelines/mine_conan.py index 1b94b749..831a0d29 100644 --- a/minecode_pipelines/pipelines/mine_conan.py +++ b/minecode_pipelines/pipelines/mine_conan.py @@ -20,44 +20,60 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from pathlib import Path -from minecode_pipelines.pipes import conan -from minecode_pipelines.pipelines import MineCodeBasePipeline +import os +from scanpipe.pipelines import Pipeline +from minecode_pipelines import pipes +from minecode_pipelines.miners import conan from scanpipe.pipes import federatedcode +MINECODE_CONAN_INDEX_REPO = "https://github.com/conan-io/conan-center-index" + +MINECODE_DATA_CONAN_REPO = os.environ.get( + "MINECODE_DATA_CONAN_REPO", "https://github.com/aboutcode-data/minecode-data-conan-test" +) -class MineConan(MineCodeBasePipeline): - """Pipeline to mine Conan packages and publish them to FederatedCode repo.""" - MINECODE_CONAN_INDEX_REPO = "https://github.com/conan-io/conan-center-index" +class MineConan(Pipeline): + """Pipeline to mine Conan packages and publish them to FederatedCode repo.""" @classmethod def steps(cls): return ( cls.check_federatedcode_eligibility, - cls.create_federatedcode_working_dir, - cls.clone_conan_index, - cls.fetch_federation_config, - cls.mine_and_publish_packageurls, - cls.delete_working_dir, + cls.clone_conan_repos, + cls.mine_and_publish_conan_package_urls, ) - def clone_conan_index(self): - """Clone the Cargo index Repo.""" - self.conan_index_repo = federatedcode.clone_repository( - repo_url=self.MINECODE_CONAN_INDEX_REPO, - clone_path=self.working_path / "conan-index", - logger=self.log, - ) + def check_federatedcode_eligibility(self): + """ + Check if the project fulfills the following criteria for + pushing the project result to FederatedCode. + """ + federatedcode.check_federatedcode_configured_and_available(logger=self.log) + + def clone_conan_repos(self): + """ + Clone the Conan-related repositories (index, data, and pipelines config) + and store their Repo objects in the corresponding instance variables. + """ + self.conan_index_repo = federatedcode.clone_repository(MINECODE_CONAN_INDEX_REPO) + self.cloned_data_repo = federatedcode.clone_repository(MINECODE_DATA_CONAN_REPO) - def packages_count(self): - base_path = Path(self.conan_index_repo.working_tree_dir) - package_dir = [p for p in base_path.iterdir() if p.is_dir() and not p.name.startswith(".")] - return sum(1 for dir in package_dir for f in dir.rglob("*") if f.is_file()) + if self.log: + self.log( + f"{MINECODE_CONAN_INDEX_REPO} repo cloned at: {self.conan_index_repo.working_dir}" + ) + self.log( + f"{MINECODE_DATA_CONAN_REPO} repo cloned at: {self.cloned_data_repo.working_dir}" + ) + + def mine_and_publish_conan_package_urls(self): + conan.mine_and_publish_conan_packageurls( + self.conan_index_repo, self.cloned_data_repo, self.log + ) - def mine_packageurls(self): - """Yield PackageURLs from Cargo index.""" - return conan.mine_conan_packageurls( - conan_index_repo=self.conan_index_repo, + def delete_cloned_repos(self): + pipes.delete_cloned_repos( + repos=[self.conan_index_repo, self.cloned_data_repo], logger=self.log, ) diff --git a/minecode_pipelines/pipelines/mine_cpan.py b/minecode_pipelines/pipelines/mine_cpan.py deleted file mode 100644 index 6add78f4..00000000 --- a/minecode_pipelines/pipelines/mine_cpan.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# http://nexb.com and https://github.com/aboutcode-org/scancode.io -# The ScanCode.io software is licensed under the Apache License version 2.0. -# Data generated with ScanCode.io is provided as-is without warranties. -# ScanCode is a trademark of nexB Inc. -# -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode.io should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# -# ScanCode.io is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/aboutcode-org/scancode.io for support and download. - - -from minecode_pipelines import pipes -from minecode_pipelines.pipes import cpan -from minecode_pipelines.pipelines import MineCodeBasePipeline -from scanpipe.pipes import federatedcode - - -class MineCpan(MineCodeBasePipeline): - """ - Mine all packageURLs from a cpan index and publish them to - a FederatedCode repo. - """ - - @classmethod - def steps(cls): - return ( - cls.check_federatedcode_eligibility, - cls.create_federatedcode_working_dir, - cls.mine_cpan_packages, - cls.fetch_federation_config, - cls.mine_and_publish_packageurls, - cls.delete_working_dir, - ) - - def mine_cpan_packages(self): - """Mine cpan package names from cpan indexes or checkpoint.""" - self.cpan_packages_path_by_name = cpan.mine_cpan_packages(logger=self.log) - - def packages_count(self): - return len(self.cpan_packages_path_by_name) - - def mine_packageurls(self): - """Get cpan packageURLs for all mined cpan package names.""" - yield from cpan.mine_and_publish_cpan_packageurls( - package_path_by_name=self.cpan_packages_path_by_name, - logger=self.log, - ) - diff --git a/minecode_pipelines/pipelines/mine_cran.py b/minecode_pipelines/pipelines/mine_cran.py index 4de4a866..b1612005 100644 --- a/minecode_pipelines/pipelines/mine_cran.py +++ b/minecode_pipelines/pipelines/mine_cran.py @@ -20,42 +20,67 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -import json -from minecode_pipelines.pipelines import MineCodeBasePipeline +import os +from scanpipe.pipelines import Pipeline +from scanpipe.pipes import federatedcode + +from minecode_pipelines import pipes +from minecode_pipelines.miners.cran import fetch_cran_db from minecode_pipelines.pipes import cran -from minecode_pipelines.pipes.cran import fetch_cran_db -class MineCran(MineCodeBasePipeline): - """Pipeline to mine CRAN R packages and publish them to FederatedCode.""" +MINECODE_DATA_CRAN_REPO = os.environ.get( + "MINECODE_DATA_CRAN_REPO", "https://github.com/aboutcode-data/minecode-data-cran-test" +) + + +class MineCran(Pipeline): + """ + Mine all packageURLs from a CRAN R index and publish them to a FederatedCode repo. + """ @classmethod def steps(cls): return ( cls.check_federatedcode_eligibility, - cls.create_federatedcode_working_dir, - cls.fetch_federation_config, - cls.download_cran_db, - cls.mine_and_publish_packageurls, - cls.delete_working_dir, + cls.setup_federatedcode_cran, + cls.mine_and_publish_cran_packageurls, + cls.cleanup_db_and_repo, ) - def download_cran_db(self): + def check_federatedcode_eligibility(self): """ - Download the full CRAN package database + Check if the project fulfills the following criteria for + pushing the project result to FederatedCode. """ - self.db_path = fetch_cran_db(working_path=self.working_path, logger=self.log) + federatedcode.check_federatedcode_configured_and_available(logger=self.log) - def packages_count(self): + def setup_federatedcode_cran(self): """ - Return the count of packages found in the downloaded CRAN JSON database. + Clone the FederatedCode CRAN repository and download the CRAN DB JSON file. """ - if not getattr(self, "db_path", None) or not self.db_path.exists(): - return None + self.cloned_data_repo = federatedcode.clone_repository(MINECODE_DATA_CRAN_REPO) + self.db_path = fetch_cran_db() + + if self.log: + self.log( + f"{MINECODE_DATA_CRAN_REPO} repo cloned at: {self.cloned_data_repo.working_dir}" + ) - with open(self.db_path, encoding="utf-8") as f: - return sum(1 for _ in json.load(f)) + def mine_and_publish_cran_packageurls(self): + """Get cran packageURLs for all mined cran package names.""" + cran.mine_and_publish_cran_packageurls( + cloned_data_repo=self.cloned_data_repo, db_path=self.db_path, logger=self.log + ) + + def cleanup_db_and_repo(self): + self.log(f"Cleaning database file at: {self.db_path}") + os.remove(self.db_path) - def mine_packageurls(self): - """Mine Cran PackageURLs from cran package database.""" - return cran.mine_cran_packageurls(db_path=self.db_path) + self.log( + f"Deleting cloned repo {MINECODE_DATA_CRAN_REPO} from: {self.cloned_data_repo.working_dir}" + ) + pipes.delete_cloned_repos( + repos=[self.cloned_data_repo], + logger=self.log, + ) diff --git a/minecode_pipelines/pipelines/mine_debian.py b/minecode_pipelines/pipelines/mine_debian.py index 52f0df83..c898eae2 100644 --- a/minecode_pipelines/pipelines/mine_debian.py +++ b/minecode_pipelines/pipelines/mine_debian.py @@ -20,72 +20,36 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +from scanpipe.pipelines import Pipeline from scanpipe.pipes import federatedcode from minecode_pipelines import pipes -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls from minecode_pipelines.pipes import debian -class MineDebian(MineCodeBasePipeline): - """Mine PackageURLs from Debian index and publish them to FederatedCode.""" - - pipeline_config_repo = "https://github.com/aboutcode-data/minecode-pipelines-config/" - checkpoint_path = "debian/checkpoints.json" - append_purls = True +class MineDebian(Pipeline): + """ + Mine all packageURLs from a Debian index and publish them to + a FederatedCode repo. + """ @classmethod def steps(cls): return ( cls.check_federatedcode_eligibility, - cls.create_federatedcode_working_dir, - cls.fetch_federation_config, - cls.fetch_checkpoint_and_debian_index, - cls.mine_and_publish_alpine_packageurls, - cls.save_check_point, - cls.delete_working_dir, + cls.collect_packages_from_debian, + cls.delete_cloned_repos, ) - def fetch_checkpoint_and_debian_index(self): - self.checkpoint_config_repo = federatedcode.clone_repository( - repo_url=self.pipeline_config_repo, - clone_path=self.working_path / "minecode-pipelines-config", - logger=self.log, - ) - checkpoint = pipes.get_checkpoint_from_file( - cloned_repo=self.checkpoint_config_repo, - path=self.checkpoint_path, - ) + def check_federatedcode_eligibility(self): + """ + Check if the project fulfills the following criteria for + pushing the project result to FederatedCode. + """ + federatedcode.check_federatedcode_configured_and_available(logger=self.log) - self.last_checkpoint = checkpoint.get("previous_debian_index_last_modified_date") - self.log(f"last_checkpoint: {self.last_checkpoint}") - self.debian_collector = debian.DebianCollector(logger=self.log) + def collect_packages_from_debian(self): + self.repos = debian.collect_packages_from_debian(logger=self.log) - def mine_and_publish_alpine_packageurls(self): - _mine_and_publish_packageurls( - packageurls=self.debian_collector.get_packages( - previous_index_last_modified_date=self.last_checkpoint, - ), - total_package_count=None, - data_cluster=self.data_cluster, - checked_out_repos=self.checked_out_repos, - working_path=self.working_path, - append_purls=self.append_purls, - commit_msg_func=self.commit_message, - logger=self.log, - ) - - def save_check_point(self): - """Save Debian checkpoint only after successful completion of PURL mining""" - from commoncode.date import get_file_mtime - - last_modified = get_file_mtime(self.debian_collector.index_location) - checkpoint = {"previous_debian_index_last_modified_date": last_modified} - self.log(f"Saving checkpoint: {checkpoint}") - pipes.update_checkpoints_in_github( - checkpoint=checkpoint, - cloned_repo=self.checkpoint_config_repo, - path=self.checkpoint_path, - logger=self.log, - ) + def delete_cloned_repos(self): + pipes.delete_cloned_repos(repos=self.repos, logger=self.log) diff --git a/minecode_pipelines/pipelines/mine_maven.py b/minecode_pipelines/pipelines/mine_maven.py index 3981d6f2..abe65484 100644 --- a/minecode_pipelines/pipelines/mine_maven.py +++ b/minecode_pipelines/pipelines/mine_maven.py @@ -20,69 +20,42 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +from scanpipe.pipelines import Pipeline from scanpipe.pipes import federatedcode from minecode_pipelines import pipes -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls from minecode_pipelines.pipes import maven -class MineMaven(MineCodeBasePipeline): - """Mine PackageURLs from maven index and publish them to FederatedCode.""" +class MineMaven(Pipeline): + """ + Create DiscoveredPackages for packages found on maven: + - input: url of maven repo + - process index + - collect purls, grouped by package + - write to files + - publish to fetchcode + - loop - pipeline_config_repo = "https://github.com/aboutcode-data/minecode-pipelines-config/" - checkpoint_path = "maven/checkpoints.json" - append_purls = True + """ @classmethod def steps(cls): return ( cls.check_federatedcode_eligibility, - cls.create_federatedcode_working_dir, - cls.fetch_federation_config, - cls.fetch_checkpoint_and_maven_index, - cls.mine_and_publish_alpine_packageurls, - cls.delete_working_dir, + cls.collect_packages_from_maven, + cls.delete_cloned_repos, ) - def fetch_checkpoint_and_maven_index(self): - self.checkpoint_config_repo = federatedcode.clone_repository( - repo_url=self.pipeline_config_repo, - clone_path=self.working_path / "minecode-pipelines-config", - logger=self.log, - ) - checkpoint = pipes.get_checkpoint_from_file( - cloned_repo=self.checkpoint_config_repo, - path=self.checkpoint_path, - ) - - last_incremental = checkpoint.get("last_incremental") - self.log(f"last_incremental: {last_incremental}") - self.maven_nexus_collector = maven.MavenNexusCollector(last_incremental=last_incremental) + def check_federatedcode_eligibility(self): + """ + Check if the project fulfills the following criteria for + pushing the project result to FederatedCode. + """ + federatedcode.check_federatedcode_configured_and_available(logger=self.log) - def mine_and_publish_alpine_packageurls(self): - _mine_and_publish_packageurls( - packageurls=self.maven_nexus_collector.get_packages(), - total_package_count=None, - data_cluster=self.data_cluster, - checked_out_repos=self.checked_out_repos, - working_path=self.working_path, - append_purls=self.append_purls, - commit_msg_func=self.commit_message, - logger=self.log, - checkpoint_func=self.save_check_point, - ) + def collect_packages_from_maven(self): + self.repos = maven.collect_packages_from_maven(logger=self.log) - def save_check_point(self): - last_incremental = self.maven_nexus_collector.index_properties.get( - "nexus.index.last-incremental" - ) - checkpoint = {"last_incremental": last_incremental} - self.log(f"Saving checkpoint: {checkpoint}") - pipes.update_checkpoints_in_github( - checkpoint=checkpoint, - cloned_repo=self.checkpoint_config_repo, - path=self.checkpoint_path, - logger=self.log, - ) + def delete_cloned_repos(self): + pipes.delete_cloned_repos(repos=self.repos, logger=self.log) diff --git a/minecode_pipelines/pipelines/mine_nuget.py b/minecode_pipelines/pipelines/mine_nuget.py index 59320a2a..7fcd87e9 100644 --- a/minecode_pipelines/pipelines/mine_nuget.py +++ b/minecode_pipelines/pipelines/mine_nuget.py @@ -46,7 +46,6 @@ def steps(cls): cls.create_federatedcode_working_dir, cls.fetch_nuget_catalog, cls.mine_nuget_package_versions, - cls.fetch_federation_config, cls.mine_and_publish_packageurls, cls.delete_working_dir, ) diff --git a/minecode_pipelines/pipelines/mine_swift.py b/minecode_pipelines/pipelines/mine_swift.py index a3438f96..74d7f437 100644 --- a/minecode_pipelines/pipelines/mine_swift.py +++ b/minecode_pipelines/pipelines/mine_swift.py @@ -20,46 +20,38 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +from scanpipe.pipelines import Pipeline from scanpipe.pipes import federatedcode +from minecode_pipelines import pipes +from minecode_pipelines.pipes.swift import mine_and_publish_swift_packageurls -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipes.swift import mine_swift_packageurls -from minecode_pipelines.pipes.swift import load_swift_package_urls - -class MineSwift(MineCodeBasePipeline): +class MineSwift(Pipeline): """ - Pipeline to mine Swift packages and publish them to FederatedCode. + Mine all packageURLs from a swift index and publish them to a FederatedCode repo. """ - swift_index_repo_url = "https://github.com/SwiftPackageIndex/PackageList" - @classmethod def steps(cls): return ( cls.check_federatedcode_eligibility, - cls.create_federatedcode_working_dir, - cls.fetch_federation_config, - cls.clone_swift_index, - cls.mine_and_publish_packageurls, - cls.delete_working_dir, + cls.mine_and_publish_swift_packageurls, + cls.delete_cloned_repos, ) - def clone_swift_index(self): - """Clone the Cargo index Repo.""" - self.swift_index_repo = federatedcode.clone_repository( - repo_url=self.swift_index_repo_url, - clone_path=self.working_path / "swift-index", - logger=self.log, - ) + def check_federatedcode_eligibility(self): + """ + Check if the project fulfills the following criteria for + pushing the project result to FederatedCode. + """ + federatedcode.check_federatedcode_configured_and_available(logger=self.log) - def packages_count(self): - return len(self.swift_packages_urls) if self.swift_packages_urls else None + def mine_and_publish_swift_packageurls(self): + """Mine swift package names from swift indexes or checkpoint.""" + self.repos = mine_and_publish_swift_packageurls(self.log) - def mine_packageurls(self): - self.swift_packages_urls = load_swift_package_urls(swift_index_repo=self.swift_index_repo) - self.log(f"Total Swift packages to process: {len(self.swift_packages_urls)}") - return mine_swift_packageurls( - packages_urls=self.swift_packages_urls, + def delete_cloned_repos(self): + pipes.delete_cloned_repos( + repos=self.repos, logger=self.log, ) diff --git a/minecode_pipelines/pipes/__init__.py b/minecode_pipelines/pipes/__init__.py index f7efa3c4..49b5207a 100644 --- a/minecode_pipelines/pipes/__init__.py +++ b/minecode_pipelines/pipes/__init__.py @@ -38,12 +38,15 @@ def fetch_checkpoint_from_github(config_repo, checkpoint_path): def get_checkpoint_from_file(cloned_repo, path): checkpoint_path = os.path.join(cloned_repo.working_dir, path) - with open(checkpoint_path) as f: - checkpoint_data = json.load(f) - return checkpoint_data or {} + try: + with open(checkpoint_path) as f: + checkpoint_data = json.load(f) + return checkpoint_data or {} + except FileNotFoundError: + return {} -def update_checkpoints_in_github(checkpoint, cloned_repo, path, logger): +def update_checkpoints_in_github(checkpoint, cloned_repo, path): from scanpipe.pipes.federatedcode import commit_and_push_changes checkpoint_path = os.path.join(cloned_repo.working_dir, path) @@ -53,7 +56,6 @@ def update_checkpoints_in_github(checkpoint, cloned_repo, path, logger): repo=cloned_repo, files_to_commit=[checkpoint_path], commit_message=commit_message, - logger=logger, ) @@ -84,10 +86,12 @@ def write_packageurls_to_file(repo, relative_datafile_path, packageurls, append= purl_file_full_path = Path(repo.working_dir) / relative_datafile_path if append and purl_file_full_path.exists(): - existing_purls = load_data_from_yaml_file(purl_file_full_path) or [] - existing_purls.extend(packageurls) - packageurls = list(set(existing_purls)) - write_data_to_yaml_file(path=purl_file_full_path, data=sorted(packageurls)) + existing_purls = load_data_from_yaml_file(purl_file_full_path) + for packageurl in packageurls: + if packageurl not in existing_purls: + existing_purls.append(packageurl) + packageurls = existing_purls + write_data_to_yaml_file(path=purl_file_full_path, data=packageurls) return relative_datafile_path @@ -202,7 +206,7 @@ def init_local_checkout(repo_name, working_path, logger): ) return { "repo": repo_obj[-1], - "file_to_commit": set(), + "file_to_commit": [], "file_processed_count": 0, "commit_count": 0, } diff --git a/minecode_pipelines/pipes/alpine.py b/minecode_pipelines/pipes/alpine.py index 009cf637..25edd74f 100644 --- a/minecode_pipelines/pipes/alpine.py +++ b/minecode_pipelines/pipes/alpine.py @@ -23,17 +23,21 @@ import base64 from shutil import rmtree -from aboutcode.pipeline import LoopProgress +from aboutcode import hashid from packagedcode.models import PackageData from packagedcode.models import Party from packageurl import PackageURL +from scanpipe.pipes import federatedcode from scanpipe.pipes.fetch import fetch_http from scanpipe.pipes.scancode import extract_archives -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode_pipelines import pipes +from minecode_pipelines import VERSION ALPINE_CHECKPOINT_PATH = "alpine/checkpoints.json" +# We are testing and storing mined packageURLs in one single repo per ecosystem for now +MINECODE_DATA_ALPINE_REPO = "https://github.com/aboutcode-data/minecode-data-alpine-test" # Number of packages PACKAGE_BATCH_SIZE = 1000 @@ -526,52 +530,86 @@ def _fetch_index(self, uri): def get_packages(self, logger=None): """Yield Package objects from alpine index""" for apkindex_url in ALPINE_LINUX_APKINDEX_URLS: - self.get_package_from_index(apkindex_url) - - def get_package_from_index(self, apkindex_url, logger=None): - _, subpath = apkindex_url.split("https://dl-cdn.alpinelinux.org/alpine/") - distro, repo, _, _ = subpath.split("/") - index = self._fetch_index(uri=apkindex_url) - extract_archives(location=index.path) - index_location = f"{index.path}-extract/APKINDEX" - with open(index_location, encoding="utf-8") as f: - for pkg in parse_apkindex(f.read()): - pd = build_package(pkg, distro=distro, repo=repo) - current_purl = PackageURL( - type=pd.type, - namespace=pd.namespace, - name=pd.name, - ) - yield current_purl, [pd.purl] - - -def mine_and_publish_alpine_packageurls( - data_cluster, - checked_out_repos, - working_path, - commit_msg_func, - logger, -): - """Yield PackageURLs from Alpine index.""" - - index_count = len(ALPINE_LINUX_APKINDEX_URLS) - progress = LoopProgress( - total_iterations=index_count, + _, subpath = apkindex_url.split("https://dl-cdn.alpinelinux.org/alpine/") + distro, repo, _, _ = subpath.split("/") + index = self._fetch_index(uri=apkindex_url) + extract_archives(location=index.path) + index_location = f"{index.path}-extract/APKINDEX" + with open(index_location, encoding="utf-8") as f: + for pkg in parse_apkindex(f.read()): + pd = build_package(pkg, distro=distro, repo=repo) + current_purl = PackageURL( + type=pd.type, + namespace=pd.namespace, + name=pd.name, + ) + yield current_purl, pd + + +def commit_message(commit_batch, total_commit_batch="many"): + from django.conf import settings + + author_name = settings.FEDERATEDCODE_GIT_SERVICE_NAME + author_email = settings.FEDERATEDCODE_GIT_SERVICE_EMAIL + tool_name = "pkg:github/aboutcode-org/scancode.io" + + return f"""\ + Collect PackageURLs from Alpine ({commit_batch}/{total_commit_batch}) + + Tool: {tool_name}@v{VERSION} + Reference: https://{settings.ALLOWED_HOSTS[0]} + + Signed-off-by: {author_name} <{author_email}> + """ + + +def collect_packages_from_alpine(files_per_commit=PACKAGE_BATCH_SIZE, logger=None): + # Clone data and config repo + data_repo = federatedcode.clone_repository( + repo_url=MINECODE_DATA_ALPINE_REPO, logger=logger, - progress_step=1, ) + config_repo = federatedcode.clone_repository( + repo_url=pipes.MINECODE_PIPELINES_CONFIG_REPO, + logger=logger, + ) + if logger: + logger(f"{MINECODE_DATA_ALPINE_REPO} repo cloned at: {data_repo.working_dir}") + logger(f"{pipes.MINECODE_PIPELINES_CONFIG_REPO} repo cloned at: {config_repo.working_dir}") - logger(f"Mine PackageURL from {index_count:,d} alpine index.") + # download and iterate through alpine indices alpine_collector = AlpineCollector() - for index in progress.iter(ALPINE_LINUX_APKINDEX_URLS): - logger(f"Mine PackageURL from {index} index.") - _mine_and_publish_packageurls( - packageurls=alpine_collector.get_package_from_index(index), - total_package_count=None, - data_cluster=data_cluster, - checked_out_repos=checked_out_repos, - working_path=working_path, - append_purls=True, - commit_msg_func=commit_msg_func, + files_to_commit = [] + commit_batch = 1 + for current_purl, package in alpine_collector.get_packages(): + # write packageURL to file + package_base_dir = hashid.get_package_base_dir(purl=current_purl) + purl_file = pipes.write_packageurls_to_file( + repo=data_repo, + base_dir=package_base_dir, + packageurls=[package.purl], + append=True, + ) + if purl_file not in files_to_commit: + files_to_commit.append(purl_file) + + if len(files_to_commit) == files_per_commit: + federatedcode.commit_and_push_changes( + commit_message=commit_message(commit_batch), + repo=data_repo, + files_to_commit=files_to_commit, + logger=logger, + ) + files_to_commit.clear() + commit_batch += 1 + + if files_to_commit: + federatedcode.commit_and_push_changes( + commit_message=commit_message(commit_batch), + repo=data_repo, + files_to_commit=files_to_commit, logger=logger, ) + + repos_to_clean = [data_repo, config_repo] + return repos_to_clean diff --git a/minecode_pipelines/pipes/cargo.py b/minecode_pipelines/pipes/cargo.py index c972bf37..11e89a4a 100644 --- a/minecode_pipelines/pipes/cargo.py +++ b/minecode_pipelines/pipes/cargo.py @@ -1,34 +1,14 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# http://nexb.com and https://github.com/aboutcode-org/scancode.io -# The ScanCode.io software is licensed under the Apache License version 2.0. -# Data generated with ScanCode.io is provided as-is without warranties. -# ScanCode is a trademark of nexB Inc. -# -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode.io should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# -# ScanCode.io is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/aboutcode-org/scancode.io for support and download. - - -import json from pathlib import Path -from aboutcode.hashid import get_core_purl + +from aboutcode import hashid from packageurl import PackageURL +from aboutcode.hashid import get_core_purl + +from minecode_pipelines.pipes import write_data_to_yaml_file -def get_cargo_packages(packages): - """Return base_purl and list of PackageURLs from cargo packages.""" +def store_cargo_packages(packages, repo): + """Collect Cargo package versions into purls and write them to the repo.""" if not packages: return @@ -46,27 +26,7 @@ def get_cargo_packages(packages): purl = PackageURL(type="cargo", name=name, version=version).to_string() updated_purls.append(purl) - return base_purl, updated_purls - - -def mine_cargo_packageurls(cargo_index_repo, logger): - """Mine Cargo PackageURLs from Crates.io package index.""" - - base_path = Path(cargo_index_repo.working_tree_dir) - package_dir = [p for p in base_path.iterdir() if p.is_dir() and not p.name.startswith(".")] - package_paths = [f for dir in package_dir for f in dir.rglob("*") if f.is_file()] - - for path in package_paths: - packages = [] - - with open(path, encoding="utf-8") as f: - for line_number, line in enumerate(f, start=1): - line = line.strip() - if not line: - continue - try: - packages.append(json.loads(line)) - except json.JSONDecodeError as e: - logger(f"Skipping invalid JSON in {path} at line {line_number}: {e}") - - yield get_cargo_packages(packages) + ppath = hashid.get_package_purls_yml_file_path(base_purl) + purl_file_full_path = Path(repo.working_dir) / ppath + write_data_to_yaml_file(path=purl_file_full_path, data=sorted(updated_purls)) + return purl_file_full_path, base_purl diff --git a/minecode_pipelines/pipes/composer.py b/minecode_pipelines/pipes/composer.py index 5eec2df9..fbc210ab 100644 --- a/minecode_pipelines/pipes/composer.py +++ b/minecode_pipelines/pipes/composer.py @@ -20,108 +20,22 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -import json -from minecode_pipelines.utils import get_temp_file -from aboutcode.hashid import get_core_purl -import requests -from packageurl import PackageURL +from datetime import datetime +from pathlib import Path +from aboutcode import hashid +from minecode_pipelines.miners.composer import get_composer_packages +from minecode_pipelines.miners.composer import load_composer_packages +from minecode_pipelines.miners.composer import get_composer_purl +from minecode_pipelines.pipes import ( + write_data_to_yaml_file, + get_checkpoint_from_file, + update_checkpoints_in_github, +) +from scanpipe.pipes.federatedcode import commit_and_push_changes from minecode_pipelines.utils import cycle_from_index, grouper -PACKAGE_BATCH_SIZE = 100 - - -def get_composer_packages(): - """ - Fetch all Composer packages from Packagist and save them to a temporary JSON file. - Response example: - { - "packageNames" ["0.0.0/composer-include-files", "0.0.0/laravel-env-shim"] - } - """ - - response = requests.get("https://packagist.org/packages/list.json") - if not response.ok: - return - - packages = response.json() - temp_file = get_temp_file("ComposerPackages", "json") - with open(temp_file, "w", encoding="utf-8") as f: - json.dump(packages, f, indent=4) - - return temp_file - - -def get_composer_purl(vendor, package): - """ - Fetch all available Package URLs (purls) for a Composer package from Packagist. - Response example: - { - "minified": "composer/2.0", - "packages": [ - { - "monolog/monolog": { - "0": { - "name": "monolog/monolog", - "version": "3.9.0" - } - } - } - ], - "security-advisories": [ - { - "advisoryId": "PKSA-dmw8-jd8k-q3c6", - "affectedVersions": ">=1.8.0,<1.12.0" - } - ] - } - get_composer_purl("monolog", "monolog") - -> ["pkg:composer/monolog/monolog@3.9.0", "pkg:composer/monolog/monolog@3.8.0", ...] - """ - purls = [] - url = f"https://repo.packagist.org/p2/{vendor}/{package}.json" - - try: - response = requests.get(url, timeout=10) - response.raise_for_status() - except requests.RequestException: - return None, purls - - data = response.json() - packages = data.get("packages", {}) - releases = packages.get(f"{vendor}/{package}", []) - - for release in releases: - version = release.get("version") - if version: - purl = PackageURL( - type="composer", - namespace=vendor, - name=package, - version=version, - ) - purls.append(purl.to_string()) - - base_purl = None - if purls: - first_purl = purls[0] - base_purl = get_core_purl(first_purl) - return base_purl, purls - - -def load_composer_packages(packages_file): - """Load and return a list of (vendor, package) tuples from a JSON file.""" - with open(packages_file, encoding="utf-8") as f: - packages_data = json.load(f) - - package_names = packages_data.get("packageNames", []) - result = [] - - for item in package_names: - if "/" in item: - vendor, package = item.split("/", 1) - result.append((vendor, package)) - - return result +PACKAGE_BATCH_SIZE = 1000 +COMPOSER_CHECKPOINT_PATH = "composer/checkpoints.json" def mine_composer_packages(): @@ -130,15 +44,55 @@ def mine_composer_packages(): return load_composer_packages(packages_file) -def mine_composer_packageurls(packages, start_index): - """Mine Composer packages from Packagist""" +def mine_and_publish_composer_purls(packages, cloned_data_repo, cloned_config_repo, logger): + """Mine Composer packages and publish their PURLs to a FederatedCode repository.""" + composer_checkpoint = get_checkpoint_from_file( + cloned_repo=cloned_config_repo, path=COMPOSER_CHECKPOINT_PATH + ) + + start_index = composer_checkpoint.get("start_index", 0) + packages_iter = cycle_from_index(packages, start_index) + for batch_index, package_batch in enumerate( grouper(n=PACKAGE_BATCH_SIZE, iterable=packages_iter) ): + purl_files = [] + purls = [] + for item in package_batch: if not item: continue vendor, package = item - yield get_composer_purl(vendor=vendor, package=package) + + updated_purls = get_composer_purl(vendor=vendor, package=package) + if not updated_purls: + continue + + base_purl = updated_purls[0] + + purl_file_full_path = Path( + cloned_data_repo.working_dir + ) / hashid.get_package_purls_yml_file_path(base_purl) + + write_data_to_yaml_file(path=purl_file_full_path, data=updated_purls) + + purl_files.append(purl_file_full_path) + purls.append(str(base_purl)) + + if purls and purl_files: + logger(f"Committing packageURLs: {', '.join(purls)}") + commit_and_push_changes( + repo=cloned_data_repo, files_to_commit=purl_files, purls=purls, logger=logger + ) + + settings_data = { + "date": str(datetime.now()), + "start_index": start_index + (batch_index + 1) * PACKAGE_BATCH_SIZE, + } + update_checkpoints_in_github( + checkpoint=settings_data, + cloned_repo=cloned_config_repo, + path=COMPOSER_CHECKPOINT_PATH, + ) diff --git a/minecode_pipelines/pipes/conan.py b/minecode_pipelines/pipes/conan.py index fda93e2a..17a03261 100644 --- a/minecode_pipelines/pipes/conan.py +++ b/minecode_pipelines/pipes/conan.py @@ -20,40 +20,24 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from pathlib import Path from packageurl import PackageURL +from pathlib import Path +from aboutcode import hashid +from minecode_pipelines.pipes import write_data_to_yaml_file -import saneyaml +def store_conan_packages(pacakge_name, versions_data, fed_repo): + """Collect Conan package versions into purls and write them to the repo.""" -def get_conan_packages(file_path, file_versions_data): - # Example: file_path = Path("repo_path/recipes/7zip/config.yml") - # - file_path.parts = ("repo_path", "recipes", "7zip", "config.yml") - # - file_path.parts[-2] = "7zip" (the package name) - if len(file_path.parts) < 2: - return None, [] - package_name = file_path.parts[-2] - base_purl = PackageURL(type="conan", name=package_name) + base_purl = PackageURL(type="conan", name=pacakge_name) updated_purls = [] - versions = file_versions_data.get("versions") or [] + versions = list(versions_data["versions"].keys()) for version in versions: - purl = PackageURL(type="conan", name=package_name, version=str(version)).to_string() + purl = PackageURL(type="conan", name=pacakge_name, version=version).to_string() updated_purls.append(purl) - return base_purl, updated_purls - - -def mine_conan_packageurls(conan_index_repo, logger): - """Mine Conan PackageURLs from package index.""" - - base_path = Path(conan_index_repo.working_dir) - for file_path in base_path.glob("recipes/**/*"): - if not file_path.name == "config.yml": - continue - with open(file_path, encoding="utf-8") as f: - versions = saneyaml.load(f) - - if not versions: - continue - yield get_conan_packages(file_path=file_path, file_versions_data=versions) + ppath = hashid.get_package_purls_yml_file_path(base_purl) + purl_file_full_path = Path(fed_repo.working_dir) / ppath + write_data_to_yaml_file(path=purl_file_full_path, data=updated_purls) + return purl_file_full_path, base_purl diff --git a/minecode_pipelines/pipes/cpan.py b/minecode_pipelines/pipes/cpan.py deleted file mode 100644 index f6844e57..00000000 --- a/minecode_pipelines/pipes/cpan.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# http://nexb.com and https://github.com/aboutcode-org/scancode.io -# The ScanCode.io software is licensed under the Apache License version 2.0. -# Data generated with ScanCode.io is provided as-is without warranties. -# ScanCode is a trademark of nexB Inc. -# -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode.io should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# -# ScanCode.io is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/aboutcode-org/scancode.io for support and download. - -from minecode_pipelines.miners.cpan import get_cpan_packages -from minecode_pipelines.miners.cpan import get_cpan_packageurls -from minecode_pipelines.miners.cpan import CPAN_REPO - -from minecode_pipelines.miners.cpan import CPAN_TYPE -from minecode_pipelines.utils import grouper - -from packageurl import PackageURL - -# If True, show full details on fetching packageURL for -# a package name present in the index -LOG_PACKAGEURL_DETAILS = False - -PACKAGE_BATCH_SIZE = 500 - - -def mine_cpan_packages(logger=None): - if logger: - logger("Getting packages from cpan index") - - package_path_by_name = get_cpan_packages(cpan_repo=CPAN_REPO, logger=logger) - - if logger: - packages_count = len(package_path_by_name.keys()) - logger(f"Mined {packages_count} packages from cpan index") - - return package_path_by_name - - -def mine_and_publish_cpan_packageurls(package_path_by_name, logger=None): - if not package_path_by_name: - return - - for package_batch in grouper(n=PACKAGE_BATCH_SIZE, iterable=package_path_by_name.keys()): - packages_mined = [] - - if logger and LOG_PACKAGEURL_DETAILS: - logger("Starting package mining for a batch of packages") - - for package_name in package_batch: - if not package_name or package_name in packages_mined: - continue - - # fetch packageURLs for package - if logger and LOG_PACKAGEURL_DETAILS: - logger(f"getting packageURLs for package: {package_name}") - - path_prefix = package_path_by_name.get(package_name) - if not path_prefix: - continue - - packageurls = get_cpan_packageurls( - name=package_name, - path_prefix=path_prefix, - logger=logger, - ) - if not packageurls: - if logger and LOG_PACKAGEURL_DETAILS: - logger(f"Package versions not present for package: {package_name}") - - # We don't want to try fetching versions for these again - packages_mined.append(package_name) - continue - - # get repo and path for package - base_purl = PackageURL(type=CPAN_TYPE, name=package_name).to_string() - if logger and LOG_PACKAGEURL_DETAILS: - logger(f"fetched packageURLs for package: {base_purl}") - purls_string = " ".join(packageurls) - logger(f"packageURLs: {purls_string}") - - packages_mined.append(package_name) - yield base_purl, packageurls diff --git a/minecode_pipelines/pipes/cran.py b/minecode_pipelines/pipes/cran.py index 861d3c52..70d9b93a 100644 --- a/minecode_pipelines/pipes/cran.py +++ b/minecode_pipelines/pipes/cran.py @@ -20,71 +20,46 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -import json -from pathlib import Path -from typing import Iterable -from typing import Tuple -from typing import List - -import requests -from packageurl import PackageURL +from aboutcode.hashid import get_package_purls_yml_file_path from aboutcode.hashid import get_core_purl +from scanpipe.pipes.federatedcode import commit_and_push_changes +from minecode_pipelines.miners.cran import extract_cran_packages +from minecode_pipelines.pipes import write_data_to_yaml_file +from minecode_pipelines.utils import grouper + +PACKAGE_BATCH_SIZE = 100 -def fetch_cran_db(working_path, logger) -> Path: +def mine_and_publish_cran_packageurls(cloned_data_repo, db_path, logger): """ - Download the CRAN package database (~250MB JSON) in a memory-efficient way. - Saves it to a file instead of loading everything into memory. + Extract CRAN packages from the database, write their package URLs (purls) to YAML, + and commit changes in batches to the given cloned repository. """ - output_path = working_path / "cran_db.json" - logger(f"Target download path: {output_path}") + packages_to_sync = list(extract_cran_packages(db_path)) - url = "https://crandb.r-pkg.org/-/all" - with requests.get(url, stream=True) as response: - response.raise_for_status() - with output_path.open("wb") as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) + for package_batch in grouper(n=PACKAGE_BATCH_SIZE, iterable=packages_to_sync): + purl_files = [] + base_purls = [] - return output_path + if logger: + logger(f"Starting package mining for a batch of {PACKAGE_BATCH_SIZE} packages") + for updated_purls in package_batch: + if not updated_purls: + continue # skip padded None values or empty -def mine_cran_packageurls(db_path: Path) -> Iterable[Tuple[str, List[str]]]: - """ - Extract package names and their versions from a CRAN DB JSON file. - Yields a tuple: (base_purl, list_of_purls) - ex: - { - "AATtools": { - "_id": "AATtools", - "_rev": "8-9ebb721d05b946f2b437b49e892c9e8c", - "name": "AATtools", - "versions": { - "0.0.1": {...}, - "0.0.2": {...}, - "0.0.3": {...} - } - } - """ - if not db_path.exists(): - raise FileNotFoundError(f"File not found: {db_path}") + first_purl = updated_purls[0] + base_purl = get_core_purl(first_purl) + purl_yaml_path = cloned_data_repo.working_dir / get_package_purls_yml_file_path( + first_purl + ) + write_data_to_yaml_file(path=purl_yaml_path, data=updated_purls) - with open(db_path, encoding="utf-8") as f: - data = json.load(f) + purl_files.append(purl_yaml_path) + base_purls.append(str(base_purl)) - for pkg_name, pkg_data in data.items(): - versions = list(pkg_data.get("versions", {}).keys()) - purls = [] - for version in versions: - purl = PackageURL( - type="cran", - name=pkg_name, - version=version, + if purl_files and base_purls: + logger(f"Committing packageURLs: {', '.join(base_purls)}") + commit_and_push_changes( + repo=cloned_data_repo, files_to_commit=purl_files, purls=base_purls, logger=logger ) - purls.append(purl.to_string()) - - base_purl = None - if purls: - first_purl = purls[0] - base_purl = get_core_purl(first_purl) - yield base_purl, purls diff --git a/minecode_pipelines/pipes/debian.py b/minecode_pipelines/pipes/debian.py index bc0514bd..9c09944b 100644 --- a/minecode_pipelines/pipes/debian.py +++ b/minecode_pipelines/pipes/debian.py @@ -21,21 +21,31 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. import gzip -import logging from datetime import datetime from shutil import rmtree -from traceback import format_exc as traceback_format_exc import debian_inspector +from aboutcode import hashid from commoncode import fileutils +from commoncode.date import get_file_mtime from packagedcode.models import PackageData from packageurl import PackageURL +from scanpipe.pipes import federatedcode from scanpipe.pipes.fetch import fetch_http +from minecode_pipelines import pipes +from minecode_pipelines import VERSION from minecode_pipelines.pipes import ls + +DEBIAN_CHECKPOINT_PATH = "debian/checkpoints.json" DEBIAN_LSLR_URL = "http://ftp.debian.org/debian/ls-lR.gz" +# We are testing and storing mined packageURLs in one single repo per ecosystem for now +MINECODE_DATA_DEBIAN_REPO = "https://github.com/aboutcode-data/minecode-data-debian-test" + +PACKAGE_BATCH_SIZE = 1000 + def is_collectible(file_name): """Return True if a `file_name` is collectible.""" @@ -72,8 +82,7 @@ class DebianCollector: Download and process a Debian ls-lR.gz file for Packages """ - def __init__(self, logger, index_location=None): - self.logger = logger + def __init__(self, index_location=None): self.downloads = [] if index_location: self.index_location = index_location @@ -107,7 +116,7 @@ def get_packages(self, previous_index_last_modified_date=None, logger=None): url_template = DEBIAN_LSLR_URL.replace("ls-lR.gz", "{path}") if previous_index_last_modified_date: previous_index_last_modified_date = datetime.strptime( - previous_index_last_modified_date, "%Y-%m-%d %H:%M:%S.%f" + previous_index_last_modified_date, "%Y-%m-%d %H:%M:%S" ) for entry in ls.parse_directory_listing(content): entry_date = None @@ -127,14 +136,7 @@ def get_packages(self, previous_index_last_modified_date=None, logger=None): continue if file_name.endswith((".deb", ".udeb", ".tar.gz", ".tar.xz", ".tar.bz2", ".tar.lzma")): - try: - name, version, arch = debian_inspector.package.get_nva(file_name) - except Exception as e: - self.logger( - f"Failed to get PURL field from: {file_name} with error {e!r}:\n{traceback_format_exc()}", - level=logging.ERROR, - ) - + name, version, arch = debian_inspector.package.get_nva(file_name) package_url = PackageURL( type="deb", namespace="debian", @@ -162,4 +164,91 @@ def get_packages(self, previous_index_last_modified_date=None, logger=None): size=entry.size, download_url=url_template.format(path=path), ) - yield versionless_purl, [packaged_data.purl] + yield versionless_purl, packaged_data + + +def commit_message(commit_batch, total_commit_batch="many"): + from django.conf import settings + + author_name = settings.FEDERATEDCODE_GIT_SERVICE_NAME + author_email = settings.FEDERATEDCODE_GIT_SERVICE_EMAIL + tool_name = "pkg:github/aboutcode-org/scancode.io" + + return f"""\ + Collect PackageURLs from Debian ({commit_batch}/{total_commit_batch}) + + Tool: {tool_name}@v{VERSION} + Reference: https://{settings.ALLOWED_HOSTS[0]} + + Signed-off-by: {author_name} <{author_email}> + """ + + +def collect_packages_from_debian(files_per_commit=PACKAGE_BATCH_SIZE, logger=None): + # Clone data and config repo + data_repo = federatedcode.clone_repository( + repo_url=MINECODE_DATA_DEBIAN_REPO, + logger=logger, + ) + config_repo = federatedcode.clone_repository( + repo_url=pipes.MINECODE_PIPELINES_CONFIG_REPO, + logger=logger, + ) + if logger: + logger(f"{MINECODE_DATA_DEBIAN_REPO} repo cloned at: {data_repo.working_dir}") + logger(f"{pipes.MINECODE_PIPELINES_CONFIG_REPO} repo cloned at: {config_repo.working_dir}") + + # get last_modified to see if we can skip files + checkpoint = pipes.get_checkpoint_from_file( + cloned_repo=config_repo, path=DEBIAN_CHECKPOINT_PATH + ) + last_modified = checkpoint.get("previous_debian_index_last_modified_date") + if logger: + logger(f"previous_debian_index_last_modified_date: {last_modified}") + + # download and iterate through debian index + debian_collector = DebianCollector() + files_to_commit = [] + commit_batch = 1 + for current_purl, package in debian_collector.get_packages( + previous_index_last_modified_date=last_modified + ): + # write packageURL to file + package_base_dir = hashid.get_package_base_dir(purl=current_purl) + purl_file = pipes.write_packageurls_to_file( + repo=data_repo, + base_dir=package_base_dir, + packageurls=[package.purl], + append=True, + ) + if purl_file not in files_to_commit: + files_to_commit.append(purl_file) + + if len(files_to_commit) == files_per_commit: + federatedcode.commit_and_push_changes( + commit_message=commit_message(commit_batch), + repo=data_repo, + files_to_commit=files_to_commit, + logger=logger, + ) + files_to_commit.clear() + commit_batch += 1 + + if files_to_commit: + federatedcode.commit_and_push_changes( + commit_message=commit_message(commit_batch), + repo=data_repo, + files_to_commit=files_to_commit, + logger=logger, + ) + + last_modified = get_file_mtime(debian_collector.index_location) + checkpoint = {"previous_debian_index_last_modified_date": last_modified} + if logger: + logger(f"checkpoint: {checkpoint}") + pipes.update_checkpoints_in_github( + checkpoint=checkpoint, cloned_repo=config_repo, path=DEBIAN_CHECKPOINT_PATH + ) + + repos_to_clean = [data_repo, config_repo] + return repos_to_clean diff --git a/minecode_pipelines/pipes/maven.py b/minecode_pipelines/pipes/maven.py index 59533ba0..c3bcbed9 100644 --- a/minecode_pipelines/pipes/maven.py +++ b/minecode_pipelines/pipes/maven.py @@ -7,24 +7,30 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -import gzip -import io -import os from collections import namedtuple from itertools import chain from shutil import rmtree +import os +import gzip +import io +import tempfile -import arrow -import javaproperties from dateutil import tz from jawa.util.utf import decode_modified_utf8 +import arrow +import javaproperties + +from aboutcode import hashid from packagedcode.maven import build_filename from packagedcode.maven import build_url from packagedcode.maven import get_urls from packagedcode.models import PackageData from packageurl import PackageURL +from scanpipe.pipes.fetch import fetch_http +from scanpipe.pipes import federatedcode - +from minecode_pipelines import pipes +from minecode_pipelines import VERSION from minecode_pipelines.pipes import java_stream TRACE = False @@ -39,6 +45,34 @@ "https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.properties" ) +MAVEN_BASE_URLS = [ + { + "url": "https://repo1.maven.org/maven2", + "checkpoint_path": "maven/repo.maven.org/checkpoints.json", + }, + { + "url": "https://repo.spring.io/artifactory/release", + "checkpoint_path": "maven/repo.spring.io-release/checkpoints.json", + }, + { + "url": "https://repo.spring.io/artifactory/milestone", + "checkpoint_path": "maven/repo.spring.io-milestone/checkpoints.json", + }, + { + "url": "https://plugins.gradle.org/m2", + "checkpoint_path": "maven/https://plugins.gradle.org/checkpoints.json", + }, + { + "url": "https://repository.apache.org/snapshots", + "checkpoint_path": "maven/repository.apache.org/checkpoints.json", + }, +] + +# We are testing and storing mined packageURLs in one single repo per ecosystem for now +MINECODE_DATA_MAVEN_REPO = "https://github.com/aboutcode-data/minecode-data-maven-test" + +PACKAGE_BATCH_SIZE = 1000 + def is_worthy_artifact(artifact): """ @@ -569,7 +603,13 @@ class MavenNexusCollector: WARNING: Processing is rather long: a full index is ~600MB. """ - def __init__(self, index_location=None, index_properties_location=None, last_incremental=None): + def __init__( + self, + maven_url=MAVEN_BASE_URL, + index_location=None, + index_properties_location=None, + last_incremental=None, + ): if index_location and last_incremental: raise Exception( "index_location and last_incremental cannot both be set at the same time. " @@ -582,7 +622,8 @@ def __init__(self, index_location=None, index_properties_location=None, last_inc if index_properties_location: self.index_properties_location = index_properties_location else: - index_property_download = self._fetch_index_properties() + uri = maven_url.rstrip("/") + "/.index/nexus-maven-repository-index.properties" + index_property_download = self._fetch_index_properties(uri=uri) self.index_properties_location = index_property_download.path if self.index_properties_location: @@ -593,8 +634,12 @@ def __init__(self, index_location=None, index_properties_location=None, last_inc if last_incremental: self.index_location = None + maven_index_increment_base_url = ( + maven_url.rstrip("/") + "/.index/nexus-maven-repository-index.{index}.gz" + ) index_increment_downloads = self._fetch_index_increments( - last_incremental=last_incremental + last_incremental=last_incremental, + maven_index_increment_base_url=maven_index_increment_base_url, ) self.index_increment_locations = [ download.path for download in index_increment_downloads @@ -603,7 +648,8 @@ def __init__(self, index_location=None, index_properties_location=None, last_inc self.index_location = index_location self.index_increment_locations = [] else: - index_download = self._fetch_index() + uri = maven_url.rstrip("/") + "/.index/nexus-maven-repository-index.gz" + index_download = self._fetch_index(uri=uri) self.index_location = index_download.path self.index_increment_locations = [] @@ -613,8 +659,6 @@ def __del__(self): rmtree(download.directory) def _fetch_http(self, uri): - from scanpipe.pipes.fetch import fetch_http - fetched = fetch_http(uri) self.downloads.append(fetched) return fetched @@ -635,7 +679,9 @@ def _fetch_index_properties(self, uri=MAVEN_INDEX_PROPERTIES_URL): index_properties = self._fetch_http(uri) return index_properties - def _fetch_index_increments(self, last_incremental): + def _fetch_index_increments( + self, last_incremental, maven_index_increment_base_url=MAVEN_INDEX_INCREMENT_BASE_URL + ): """ Fetch maven index increments, starting past `last_incremental`, and return a list of Downloads with information about where they were saved. @@ -645,7 +691,7 @@ def _fetch_index_increments(self, last_incremental): if increment_index <= last_incremental: continue if key.startswith("nexus.index.incremental"): - index_increment_url = MAVEN_INDEX_INCREMENT_BASE_URL.format(index=increment_index) + index_increment_url = maven_index_increment_base_url.format(index=increment_index) index_increment = self._fetch_http(index_increment_url) index_increment_downloads.append(index_increment) return index_increment_downloads @@ -715,7 +761,7 @@ def _get_packages(self, content=None): name=artifact_id, version=version, ) - yield current_purl, [package.purl] + yield current_purl, package def _get_packages_from_index_increments(self): for index_increment in self.index_increment_locations: @@ -728,3 +774,102 @@ def get_packages(self): else: packages = self._get_packages(content=self.index_location) return packages + + +def commit_message(commit_batch, total_commit_batch="many"): + from django.conf import settings + + author_name = settings.FEDERATEDCODE_GIT_SERVICE_NAME + author_email = settings.FEDERATEDCODE_GIT_SERVICE_EMAIL + tool_name = "pkg:github/aboutcode-org/scancode.io" + + return f"""\ + Collect PackageURLs from Maven ({commit_batch}/{total_commit_batch}) + + Tool: {tool_name}@v{VERSION} + Reference: https://{settings.ALLOWED_HOSTS[0]} + + Signed-off-by: {author_name} <{author_email}> + """ + + +def collect_packages_from_maven(files_per_commit=PACKAGE_BATCH_SIZE, logger=None): + tmp_dir_data = tempfile.mkdtemp() + tmp_dir_config = tempfile.mkdtemp() + # Clone data and config repo + data_repo = federatedcode.clone_repository( + repo_url=MINECODE_DATA_MAVEN_REPO, + clone_path=tmp_dir_data, + logger=logger, + ) + + config_repo = federatedcode.clone_repository( + repo_url=pipes.MINECODE_PIPELINES_CONFIG_REPO, + clone_path=tmp_dir_config, + logger=logger, + ) + if logger: + logger(f"{MINECODE_DATA_MAVEN_REPO} repo cloned at: {data_repo.working_dir}") + logger(f"{pipes.MINECODE_PIPELINES_CONFIG_REPO} repo cloned at: {config_repo.working_dir}") + + for base in MAVEN_BASE_URLS: + if logger: + logger(f"Collecting packages from Maven base repo: {base['url']}") + # get last_incremental to see if we can start from incrementals + checkpoint = pipes.get_checkpoint_from_file( + cloned_repo=config_repo, path=base["checkpoint_path"] + ) + last_incremental = checkpoint.get("last_incremental") + if logger: + logger(f"last_incremental: {last_incremental}") + + # download and iterate through maven nexus index + maven_nexus_collector = MavenNexusCollector( + maven_url=base["url"], last_incremental=last_incremental + ) + files_to_commit = [] + commit_batch = 1 + for current_purl, package in maven_nexus_collector.get_packages(): + # write packageURL to file + package_base_dir = hashid.get_package_base_dir(purl=current_purl) + relative_datafile_path = os.path.join(package_base_dir, hashid.PURLS_FILENAME) + purl_file = pipes.write_packageurls_to_file( + repo=data_repo, + relative_datafile_path=relative_datafile_path, + packageurls=[package.purl], + append=True, + ) + if purl_file not in files_to_commit: + files_to_commit.append(purl_file) + + if len(files_to_commit) == files_per_commit: + federatedcode.commit_and_push_changes( + commit_message=commit_message(commit_batch), + repo=data_repo, + files_to_commit=files_to_commit, + logger=logger, + ) + files_to_commit.clear() + commit_batch += 1 + + if files_to_commit: + federatedcode.commit_and_push_changes( + commit_message=commit_message(commit_batch), + repo=data_repo, + files_to_commit=files_to_commit, + logger=logger, + ) + + # update last_incremental so we can pick up from the proper place next time + last_incremental = maven_nexus_collector.index_properties.get( + "nexus.index.last-incremental" + ) + checkpoint = {"last_incremental": last_incremental} + if logger: + logger(f"checkpoint: {checkpoint}") + pipes.update_checkpoints_in_github( + checkpoint=checkpoint, cloned_repo=config_repo, path=base["checkpoint_path"] + ) + + repos_to_clean = [data_repo, config_repo] + return repos_to_clean diff --git a/minecode_pipelines/pipes/swift.py b/minecode_pipelines/pipes/swift.py index 47566681..9755d2c9 100644 --- a/minecode_pipelines/pipes/swift.py +++ b/minecode_pipelines/pipes/swift.py @@ -21,43 +21,38 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. import json +import os +from datetime import datetime from pathlib import Path +from aboutcode import hashid from packageurl import PackageURL -import shutil -import subprocess -from urllib.parse import urlparse +from minecode_pipelines.miners.swift import fetch_git_tags_raw +from minecode_pipelines.miners.swift import get_tags_and_commits_from_git_output +from minecode_pipelines.miners.swift import split_org_repo +from minecode_pipelines.pipes import update_checkpoints_in_github +from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO +from minecode_pipelines.pipes import write_data_to_yaml_file -def mine_swift_packageurls(packages_urls, logger): - """Mine Swift PackageURLs from package index.""" +from minecode_pipelines.pipes import get_checkpoint_from_file +from scanpipe.pipes.federatedcode import clone_repository - for package_repo_url in packages_urls: - logger(f"Processing package repo URL: {package_repo_url}") - git_ls_remote = fetch_git_tags_raw(package_repo_url, 60, logger) - if not git_ls_remote: - continue +from scanpipe.pipes.federatedcode import commit_and_push_changes +from minecode_pipelines.utils import cycle_from_index - tags_and_commits = get_tags_and_commits_from_git_output(git_ls_remote) - if not tags_and_commits: - continue +PACKAGE_BATCH_SIZE = 1000 +SWIFT_CHECKPOINT_PATH = "swift/checkpoints.json" - yield generate_package_urls( - package_repo_url=package_repo_url, tags_and_commits=tags_and_commits, logger=logger - ) +MINECODE_DATA_SWIFT_REPO = os.environ.get( + "MINECODE_DATA_SWIFT_REPO", "https://github.com/aboutcode-data/minecode-data-swift-test" +) +MINECODE_SWIFT_INDEX_REPO = "https://github.com/SwiftPackageIndex/PackageList" -def load_swift_package_urls(swift_index_repo): - packages_path = Path(swift_index_repo.working_dir) / "packages.json" - with open(packages_path) as f: - packages_urls = json.load(f) - return packages_urls - - -def generate_package_urls(package_repo_url, tags_and_commits, logger): - org, name = split_org_repo(package_repo_url, logger) - if not org or not name: - return None, [] +def store_swift_packages(package_repo_url, tags_and_commits, cloned_data_repo): + """Collect Swift package versions into purls and write them to the repo.""" + org, name = split_org_repo(package_repo_url) org = "github.com/" + org base_purl = PackageURL(type="swift", namespace=org, name=name) updated_purls = [] @@ -74,91 +69,103 @@ def generate_package_urls(package_repo_url, tags_and_commits, logger): if purl: updated_purls.append(purl) - logger( - f"Generated {len(updated_purls)} and base PURL: {base_purl} PackageURLs for {package_repo_url}" + + purl_yaml_path = cloned_data_repo.working_dir / hashid.get_package_purls_yml_file_path( + base_purl ) - return base_purl, updated_purls + write_data_to_yaml_file(path=purl_yaml_path, data=updated_purls) + return purl_yaml_path, base_purl -def is_safe_repo_url(repo_url: str) -> bool: - """Return True if the URL is HTTPS GitHub with .git suffix or has at least two path segments.""" - parsed = urlparse(repo_url) - return ( - parsed.scheme == "https" and parsed.netloc == "github.com" and parsed.path.endswith(".git") +def mine_and_publish_swift_packageurls(logger): + """ + Clone Swift-related repositories, process Swift packages, and publish their + Package URLs (purls) to the data repository. + + This function: + 1. Clones the Swift index, data, and pipelines config repositories. + 2. Loads the list of Swift package repositories from `packages.json`. + 3. Iterates over each package, fetching tags/commits and generating purls. + 4. Commits and pushes purl files to the data repository in batches. + 5. Updates checkpoint information in the config repository to track progress. + + logger (callable): Optional logging function for status updates. + Returns: list: A list of cloned repository objects in the order: + [swift_index_repo, cloned_data_repo, cloned_config_repo] + """ + + swift_index_repo = clone_repository(MINECODE_SWIFT_INDEX_REPO) + cloned_data_repo = clone_repository(MINECODE_DATA_SWIFT_REPO) + cloned_config_repo = clone_repository(MINECODE_PIPELINES_CONFIG_REPO) + + if logger: + logger(f"{MINECODE_SWIFT_INDEX_REPO} repo cloned at: {swift_index_repo.working_dir}") + logger(f"{MINECODE_DATA_SWIFT_REPO} repo cloned at: {cloned_data_repo.working_dir}") + logger(f"{MINECODE_PIPELINES_CONFIG_REPO} repo cloned at: {cloned_config_repo.working_dir}") + + packages_path = Path(swift_index_repo.working_dir) / "packages.json" + with open(packages_path) as f: + packages_urls = json.load(f) + + counter = 0 + purl_files = [] + purls = [] + + swift_checkpoint = get_checkpoint_from_file( + cloned_repo=cloned_config_repo, path=SWIFT_CHECKPOINT_PATH ) + start_index = swift_checkpoint.get("start_index", 0) + + if logger: + logger(f"Processing total files: {len(packages_urls)}") + + for idx, package_repo_url in enumerate(cycle_from_index(packages_urls, start_index)): + git_ls_remote = fetch_git_tags_raw(package_repo_url, 60, logger) + if not git_ls_remote: + continue + + tags_and_commits = get_tags_and_commits_from_git_output(git_ls_remote) + if not tags_and_commits: + continue -def fetch_git_tags_raw(repo_url: str, timeout: int = 60, logger=None) -> str | None: - """Run `git ls-remote` on a GitHub repo and return raw output, or None on error.""" - git_executable = shutil.which("git") - if git_executable is None: - logger("Git executable not found in PATH") - return None - - if not repo_url: - logger("No repository URL provided") - return None - - if not is_safe_repo_url(repo_url): - logger(f"Unsafe repository URL: {repo_url}") - return None - - try: - result = subprocess.run( # NOQA - [git_executable, "ls-remote", repo_url], - capture_output=True, - text=True, - check=True, - timeout=timeout, + purl_file, base_purl = store_swift_packages( + package_repo_url, tags_and_commits, cloned_data_repo ) - return result.stdout.strip() - except subprocess.CalledProcessError as e: - logger(f"Failed to fetch tags for {repo_url}: {e}") - except subprocess.TimeoutExpired: - logger(f"Timeout fetching tags for {repo_url}") - return None + purl_files.append(purl_file) + purls.append(str(base_purl)) + counter += 1 + + if counter >= PACKAGE_BATCH_SIZE: + if purls and purl_files: + logger(f"Committing packageURLs: {', '.join(purls)}") + commit_and_push_changes( + repo=cloned_data_repo, files_to_commit=purl_files, purls=purls, logger=logger + ) + + purl_files = [] + purls = [] + counter = 0 + + if start_index == idx: + continue + + settings_data = { + "date": str(datetime.now()), + "start_index": idx, + } + + update_checkpoints_in_github( + checkpoint=settings_data, + cloned_repo=cloned_config_repo, + path=SWIFT_CHECKPOINT_PATH, + ) + + if purls and purl_files: + logger(f"Committing packageURLs: {', '.join(purls)}") + commit_and_push_changes( + repo=cloned_data_repo, files_to_commit=purl_files, purls=purls, logger=logger + ) -# FIXME duplicated with miners github -def split_org_repo(url_like, logger): - """ - Given a URL-like string to a GitHub repo or a repo name as in org/name, - split and return the org and name. - - For example: - >>> split_org_repo('foo/bar') - ('foo', 'bar') - >>> split_org_repo('https://api.github.com/repos/foo/bar/') - ('foo', 'bar') - >>> split_org_repo('github.com/foo/bar/') - ('foo', 'bar') - >>> split_org_repo('git://github.com/foo/bar.git') - ('foo', 'bar') - """ - segments = [s.strip() for s in url_like.split("/") if s.strip()] - if not len(segments) >= 2: - logger(f"Could not parse org and name from URL-like: {url_like}") - return None, None - org = segments[-2] - name = segments[-1] - if name.endswith(".git"): - name, _, _ = name.rpartition(".git") - return org, name - - -def get_tags_and_commits_from_git_output(git_ls_remote): - """ - Yield tuples of (tag, commit), given a git ls-remote output - """ - tags_and_commits = [] - for line in git_ls_remote.split("\n"): - # line: kjwfgeklngelkfjofjeo123 refs/tags/1.2.3 - line_segments = line.split("\t") - # segments: ["kjwfgeklngelkfjofjeo123", "refs/tags/1.2.3"] - if len(line_segments) > 1 and ( - line_segments[1].startswith("refs/tags/") or line_segments[1] == "HEAD" - ): - commit = line_segments[0] - tag = line_segments[1].replace("refs/tags/", "") - tags_and_commits.append((tag, commit)) - return tags_and_commits + return [swift_index_repo, cloned_data_repo, cloned_config_repo] diff --git a/minecode_pipelines/tests/pipes/test_cargo.py b/minecode_pipelines/tests/pipes/test_cargo.py index 76df7d7d..742f428a 100644 --- a/minecode_pipelines/tests/pipes/test_cargo.py +++ b/minecode_pipelines/tests/pipes/test_cargo.py @@ -8,19 +8,22 @@ # import json - +import tempfile from pathlib import Path - +from unittest import mock +from unittest.mock import Mock, patch import saneyaml -from unittest import TestCase +from django.test import TestCase -from minecode_pipelines.pipes.cargo import get_cargo_packages +from minecode_pipelines.pipes import write_data_to_yaml_file +from minecode_pipelines.pipes.cargo import store_cargo_packages DATA_DIR = Path(__file__).parent.parent / "test_data" / "cargo" class CargoPipelineTests(TestCase): - def test_collect_packages_from_cargo_calls_write(self): + @patch("minecode_pipelines.pipes.cargo.write_data_to_yaml_file") + def test_collect_packages_from_cargo_calls_write(self, mock_write): packages_file = DATA_DIR / "c5store" expected_file = DATA_DIR / "c5store-expected.yaml" @@ -33,7 +36,50 @@ def test_collect_packages_from_cargo_calls_write(self): with open(expected_file, encoding="utf-8") as f: expected = saneyaml.load(f) - base, purls = get_cargo_packages(packages) + with tempfile.TemporaryDirectory() as tmpdir: + repo = Mock() + repo.working_dir = tmpdir + + store_cargo_packages(packages, repo) + + mock_write.assert_called_once() + args, kwargs = mock_write.call_args + base_purl, written_packages = kwargs["path"], kwargs["data"] + + expected_base_purl = ( + Path(tmpdir) / "aboutcode-packages-cargo-0" / "cargo" / "c5store" / "purls.yml" + ) + + self.assertEqual(str(base_purl), str(expected_base_purl)) + self.assertEqual(written_packages, expected) + + def _assert_purls_written(self, purls): + with tempfile.TemporaryDirectory() as tmpdir: + repo_dir = Path(tmpdir) + + mock_repo = mock.MagicMock() + mock_repo.working_dir = str(repo_dir) + mock_repo.index.add = mock.MagicMock() + + purls_file = repo_dir / "purls.yaml" + + write_data_to_yaml_file(purls_file, purls) + + self.assertTrue(purls_file.exists()) + + with open(purls_file, encoding="utf-8") as f: + content = saneyaml.load(f) + + self.assertEqual(content, purls) + + def test_add_purl_result_with_mock_repo(self): + self._assert_purls_written( + [{"purl": "pkg:pypi/django@4.2.0"}, {"purl": "pkg:pypi/django@4.3.0"}] + ) + + def test_add_empty_purl_result_with_mock_repo(self): + self._assert_purls_written([]) - self.assertEqual(str(base), "pkg:cargo/c5store") - self.assertEqual(purls, expected) + def test_add_invalid_purl_with_mock_repo(self): + # invalid but still written as empty file + self._assert_purls_written([{"purl": "pkg:pypi/django"}]) diff --git a/minecode_pipelines/tests/pipes/test_composer.py b/minecode_pipelines/tests/pipes/test_composer.py index 6d809802..42976f65 100644 --- a/minecode_pipelines/tests/pipes/test_composer.py +++ b/minecode_pipelines/tests/pipes/test_composer.py @@ -12,9 +12,9 @@ from unittest.mock import patch, MagicMock from django.test import SimpleTestCase -from minecode_pipelines.pipes.composer import get_composer_packages -from minecode_pipelines.pipes.composer import load_composer_packages -from minecode_pipelines.pipes.composer import get_composer_purl +from minecode_pipelines.miners.composer import get_composer_packages +from minecode_pipelines.miners.composer import load_composer_packages +from minecode_pipelines.miners.composer import get_composer_purl DATA_DIR = Path(__file__).parent.parent / "test_data" / "composer" @@ -51,8 +51,7 @@ def test_generate_purls_from_composer(self, mock_get): all_purls = [] for vendor, package in packages: - base_purl, purls = get_composer_purl(vendor, package) - assert str(base_purl) == "pkg:composer/monolog/monolog" + purls = get_composer_purl(vendor, package) all_purls.extend(purls) assert len(all_purls) == 85 diff --git a/minecode_pipelines/tests/pipes/test_conan.py b/minecode_pipelines/tests/pipes/test_conan.py index 044fd1ee..2a9e8281 100644 --- a/minecode_pipelines/tests/pipes/test_conan.py +++ b/minecode_pipelines/tests/pipes/test_conan.py @@ -7,20 +7,25 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import tempfile from pathlib import Path +from unittest import mock +from unittest.mock import Mock, patch import saneyaml import yaml from django.test import TestCase -from minecode_pipelines.pipes.conan import get_conan_packages +from minecode_pipelines.pipes import write_data_to_yaml_file +from minecode_pipelines.pipes.conan import store_conan_packages DATA_DIR = Path(__file__).parent.parent / "test_data" / "conan" class ConanPipelineTests(TestCase): - def test_collect_packages_from_conan_calls_write(self, mock_write): - packages_file = DATA_DIR / "cairo" / "cairo-config.yml" + @patch("minecode_pipelines.pipes.conan.write_data_to_yaml_file") + def test_collect_packages_from_cargo_calls_write(self, mock_write): + packages_file = DATA_DIR / "cairo-config.yml" expected_file = DATA_DIR / "expected-cairo-purls.yml" with open(packages_file, encoding="utf-8") as f: @@ -29,6 +34,51 @@ def test_collect_packages_from_conan_calls_write(self, mock_write): with open(expected_file, encoding="utf-8") as f: expected = saneyaml.load(f) - base, purls = get_conan_packages(packages_file, versions_data) - self.assertEqual(purls, expected) - self.assertEqual(str(base), "pkg:conan/cairo") + with tempfile.TemporaryDirectory() as tmpdir: + repo = Mock() + repo.working_dir = tmpdir + + store_conan_packages("cairo", versions_data, repo) + + mock_write.assert_called_once() + args, kwargs = mock_write.call_args + base_purl, written_packages = kwargs["path"], kwargs["data"] + + expected_base_purl = ( + Path(tmpdir) / "aboutcode-packages-conan-0" / "conan" / "cairo" / "purls.yml" + ) + + self.assertEqual(str(base_purl), str(expected_base_purl)) + self.assertEqual(written_packages, expected) + + def _assert_purls_written(self, purls): + with tempfile.TemporaryDirectory() as tmpdir: + repo_dir = Path(tmpdir) + + mock_repo = mock.MagicMock() + mock_repo.working_dir = str(repo_dir) + mock_repo.index.add = mock.MagicMock() + + purls_file = repo_dir / "purls.yaml" + + write_data_to_yaml_file(purls_file, purls) + + self.assertTrue(purls_file.exists()) + + with open(purls_file, encoding="utf-8") as f: + content = saneyaml.load(f) + + self.assertEqual(content, purls) + + def test_add_purl_result_with_mock_repo(self): + purls = [ + {"purl": "pkg:conan/cairo@1.18.0"}, + {"purl": "pkg:conan/cairo@1.17.8"}, + {"purl": "pkg:conan/cairo@1.17.6"}, + {"purl": "pkg:conan/cairo@1.17.4"}, + ] + + self._assert_purls_written(purls) + + def test_add_empty_purl_result_with_mock_repo(self): + self._assert_purls_written([]) diff --git a/minecode_pipelines/tests/pipes/test_cran.py b/minecode_pipelines/tests/pipes/test_cran.py index 16fb0d83..ef194d71 100644 --- a/minecode_pipelines/tests/pipes/test_cran.py +++ b/minecode_pipelines/tests/pipes/test_cran.py @@ -10,33 +10,32 @@ import saneyaml from pathlib import Path from unittest import TestCase +from minecode_pipelines.miners.cran import extract_cran_packages -from minecode_pipelines.pipes.cran import mine_cran_packageurls DATA_DIR = Path(__file__).parent.parent / "test_data" / "cran" + class CranPipelineTests(TestCase): - def test_mine_cran_packageurls_from_testdata(self): + def test_extract_cran_packages_from_testdata(self): """ - Ensure mine_cran_packageurls correctly parses the CRAN database + Ensure extract_cran_packages correctly parses the CRAN database and produces results identical to the expected YAML files. """ db_file = DATA_DIR / "cran_db.json" - results = list(mine_cran_packageurls(db_file)) + results = list(extract_cran_packages(db_file)) expected_files = [ DATA_DIR / "expected_abbreviate.yaml", DATA_DIR / "expected_abc.data.yaml", DATA_DIR / "expected_abc.yaml", ] - expected_base_purls = ["pkg:cran/abbreviate", "pkg:cran/abc", "pkg:cran/abc.data"] + assert len(results) == len(expected_files) - for result, expected_base_purl, expected_file in zip(results, expected_base_purls, expected_files): + for result, expected_file in zip(results, expected_files): with open(expected_file, encoding="utf-8") as f: - expected_purls = saneyaml.load(f) + expected = saneyaml.load(f) - base_purl, purls = result - assert str(base_purl) == expected_base_purl - assert purls == expected_purls + assert result == expected diff --git a/minecode_pipelines/tests/pipes/test_swift.py b/minecode_pipelines/tests/pipes/test_swift.py index ea1eafee..cf0b6ce9 100644 --- a/minecode_pipelines/tests/pipes/test_swift.py +++ b/minecode_pipelines/tests/pipes/test_swift.py @@ -7,40 +7,50 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import tempfile from pathlib import Path from unittest import TestCase +from unittest.mock import Mock, patch import saneyaml from minecode_pipelines.pipes.swift import ( + store_swift_packages, get_tags_and_commits_from_git_output, - generate_package_urls, ) DATA_DIR = Path(__file__).parent.parent / "data" / "swift" -def logger(msg): - print(msg) - - class SwiftPipelineTests(TestCase): - def _run_package_test(self, package_repo_url, commits_tags_file, expected_file): + def _run_package_test( + self, mock_write, package_repo_url, commits_tags_file, expected_file, expected_path_parts + ): + # Load test input and expected output with open(commits_tags_file, encoding="utf-8") as f: git_ls_remote = f.read() - with open(expected_file, encoding="utf-8") as f: expected = saneyaml.load(f) - tags_and_commits = get_tags_and_commits_from_git_output(git_ls_remote) + # Create a temporary working directory for the repo + with tempfile.TemporaryDirectory() as tmpdir: + repo = Mock() + repo.working_dir = tmpdir - base_purl, generated_purls = generate_package_urls( - package_repo_url=package_repo_url, - tags_and_commits=tags_and_commits, - logger=logger, - ) + # Execute function under test + tags_and_commits = get_tags_and_commits_from_git_output(git_ls_remote) + store_swift_packages(package_repo_url, tags_and_commits, repo) + + # Verify function call + mock_write.assert_called_once() + _, kwargs = mock_write.call_args + base_purl, written_packages = kwargs["path"], kwargs["data"] - assert base_purl.to_string() == expected["base_purl"] - assert sorted(generated_purls) == sorted(expected["purls"]) + # Expected file path + expected_base_purl = Path(tmpdir).joinpath(*expected_path_parts) + self.assertEqual(str(base_purl), str(expected_base_purl)) + self.assertEqual(written_packages, expected) + + @patch("minecode_pipelines.pipes.swift.write_data_to_yaml_file") def test_swift_safe_collection_access(self, mock_write): self._run_package_test( mock_write, @@ -57,8 +67,10 @@ def test_swift_safe_collection_access(self, mock_write): ], ) - def test_human_string(self): + @patch("minecode_pipelines.pipes.swift.write_data_to_yaml_file") + def test_human_string(self, mock_write): self._run_package_test( + mock_write, package_repo_url="https://github.com/zonble/HumanString.git", commits_tags_file=DATA_DIR / "commits_tags2.txt", expected_file=DATA_DIR / "expected2.yaml", @@ -72,8 +84,10 @@ def test_human_string(self): ], ) - def test_swift_financial(self): + @patch("minecode_pipelines.pipes.swift.write_data_to_yaml_file") + def test_swift_financial(self, mock_write): self._run_package_test( + mock_write, package_repo_url="https://github.com/zrluety/SwiftFinancial.git", commits_tags_file=DATA_DIR / "commits_tags3.txt", expected_file=DATA_DIR / "expected3.yaml", @@ -87,8 +101,10 @@ def test_swift_financial(self): ], ) - def test_swift_xcf_sodium(self): + @patch("minecode_pipelines.pipes.swift.write_data_to_yaml_file") + def test_swift_xcf_sodium(self, mock_write): self._run_package_test( + mock_write, package_repo_url="https://github.com/0xacdc/XCFSodium", commits_tags_file=DATA_DIR / "commits_tags4.txt", expected_file=DATA_DIR / "expected4.yaml", diff --git a/minecode_pipelines/tests/test_data/composer/package_details.json b/minecode_pipelines/tests/test_data/composer/package_details.json index a7b3f42c..facc6851 100644 --- a/minecode_pipelines/tests/test_data/composer/package_details.json +++ b/minecode_pipelines/tests/test_data/composer/package_details.json @@ -1,2626 +1,87 @@ -{ - "minified": "composer/2.0", - "packages": { - "monolog/monolog": [ - { - "name": "monolog/monolog", - "description": "Sends your logs to files, sockets, inboxes, databases and various web services", - "keywords": [ - "log", - "logging", - "psr-3" - ], - "homepage": "https://github.com/Seldaek/monolog", - "version": "3.9.0", - "version_normalized": "3.9.0.0", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Jordi Boggiano", - "email": "j.boggiano@seld.be", - "homepage": "https://seld.be" - } - ], - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "10d85740180ecba7896c87e06a166e0c95a0e3b6" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/10d85740180ecba7896c87e06a166e0c95a0e3b6", - "type": "zip", - "shasum": "", - "reference": "10d85740180ecba7896c87e06a166e0c95a0e3b6" - }, - "type": "library", - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.9.0" - }, - "funding": [ - { - "url": "https://github.com/Seldaek", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/monolog/monolog", - "type": "tidelift" - } - ], - "time": "2025-03-24T10:02:05+00:00", - "autoload": { - "psr-4": { - "Monolog\\": "src/Monolog" - } - }, - "extra": { - "branch-alias": { - "dev-main": "3.x-dev" - } - }, - "require": { - "php": ">=8.1", - "psr/log": "^2.0 || ^3.0" - }, - "require-dev": { - "ext-json": "*", - "aws/aws-sdk-php": "^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7 || ^8", - "graylog2/gelf-php": "^1.4.2 || ^2.0", - "guzzlehttp/guzzle": "^7.4.5", - "guzzlehttp/psr7": "^2.2", - "mongodb/mongodb": "^1.8", - "php-amqplib/php-amqplib": "~2.4 || ^3", - "php-console/php-console": "^3.1.8", - "phpstan/phpstan": "^2", - "phpstan/phpstan-deprecation-rules": "^2", - "phpstan/phpstan-strict-rules": "^2", - "phpunit/phpunit": "^10.5.17 || ^11.0.7", - "predis/predis": "^1.1 || ^2", - "rollbar/rollbar": "^4.0", - "ruflin/elastica": "^7 || ^8", - "symfony/mailer": "^5.4 || ^6", - "symfony/mime": "^5.4 || ^6" - }, - "suggest": { - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "elasticsearch/elasticsearch": "Allow sending log messages to an Elasticsearch server via official client", - "php-amqplib/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongodb": "Allow sending log messages to a MongoDB server (via driver)", - "mongodb/mongodb": "Allow sending log messages to a MongoDB server (via library)", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "rollbar/rollbar": "Allow sending log messages to Rollbar", - "ext-mbstring": "Allow to work properly with unicode symbols", - "ext-sockets": "Allow sending log messages to a Syslog server (via UDP driver)", - "ext-curl": "Required to send log messages using the IFTTTHandler, the LogglyHandler, the SendGridHandler, the SlackWebhookHandler or the TelegramBotHandler", - "ext-openssl": "Required to send log messages using SSL" - }, - "provide": { - "psr/log-implementation": "3.0.0" - } - }, - { - "version": "3.8.1", - "version_normalized": "3.8.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "aef6ee73a77a66e404dd6540934a9ef1b3c855b4" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/aef6ee73a77a66e404dd6540934a9ef1b3c855b4", - "type": "zip", - "shasum": "", - "reference": "aef6ee73a77a66e404dd6540934a9ef1b3c855b4" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.8.1" - }, - "time": "2024-12-05T17:15:07+00:00" - }, - { - "version": "3.8.0", - "version_normalized": "3.8.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "32e515fdc02cdafbe4593e30a9350d486b125b67" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/32e515fdc02cdafbe4593e30a9350d486b125b67", - "type": "zip", - "shasum": "", - "reference": "32e515fdc02cdafbe4593e30a9350d486b125b67" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.8.0" - }, - "time": "2024-11-12T13:57:08+00:00" - }, - { - "version": "3.7.0", - "version_normalized": "3.7.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "f4393b648b78a5408747de94fca38beb5f7e9ef8" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/f4393b648b78a5408747de94fca38beb5f7e9ef8", - "type": "zip", - "shasum": "", - "reference": "f4393b648b78a5408747de94fca38beb5f7e9ef8" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.7.0" - }, - "time": "2024-06-28T09:40:51+00:00", - "require-dev": { - "ext-json": "*", - "aws/aws-sdk-php": "^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7 || ^8", - "graylog2/gelf-php": "^1.4.2 || ^2.0", - "guzzlehttp/guzzle": "^7.4.5", - "guzzlehttp/psr7": "^2.2", - "mongodb/mongodb": "^1.8", - "php-amqplib/php-amqplib": "~2.4 || ^3", - "phpstan/phpstan": "^1.9", - "phpstan/phpstan-deprecation-rules": "^1.0", - "phpstan/phpstan-strict-rules": "^1.4", - "phpunit/phpunit": "^10.5.17", - "predis/predis": "^1.1 || ^2", - "ruflin/elastica": "^7", - "symfony/mailer": "^5.4 || ^6", - "symfony/mime": "^5.4 || ^6" - } - }, - { - "version": "3.6.0", - "version_normalized": "3.6.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "4b18b21a5527a3d5ffdac2fd35d3ab25a9597654" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/4b18b21a5527a3d5ffdac2fd35d3ab25a9597654", - "type": "zip", - "shasum": "", - "reference": "4b18b21a5527a3d5ffdac2fd35d3ab25a9597654" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.6.0" - }, - "time": "2024-04-12T21:02:21+00:00" - }, - { - "version": "3.5.0", - "version_normalized": "3.5.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "c915e2634718dbc8a4a15c61b0e62e7a44e14448" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/c915e2634718dbc8a4a15c61b0e62e7a44e14448", - "type": "zip", - "shasum": "", - "reference": "c915e2634718dbc8a4a15c61b0e62e7a44e14448" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.5.0" - }, - "time": "2023-10-27T15:32:31+00:00", - "require-dev": { - "ext-json": "*", - "aws/aws-sdk-php": "^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7 || ^8", - "graylog2/gelf-php": "^1.4.2 || ^2.0", - "guzzlehttp/guzzle": "^7.4.5", - "guzzlehttp/psr7": "^2.2", - "mongodb/mongodb": "^1.8", - "php-amqplib/php-amqplib": "~2.4 || ^3", - "phpstan/phpstan": "^1.9", - "phpstan/phpstan-deprecation-rules": "^1.0", - "phpstan/phpstan-strict-rules": "^1.4", - "phpunit/phpunit": "^10.1", - "predis/predis": "^1.1 || ^2", - "ruflin/elastica": "^7", - "symfony/mailer": "^5.4 || ^6", - "symfony/mime": "^5.4 || ^6" - } - }, - { - "version": "3.4.0", - "version_normalized": "3.4.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "e2392369686d420ca32df3803de28b5d6f76867d" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/e2392369686d420ca32df3803de28b5d6f76867d", - "type": "zip", - "shasum": "", - "reference": "e2392369686d420ca32df3803de28b5d6f76867d" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.4.0" - }, - "time": "2023-06-21T08:46:11+00:00" - }, - { - "version": "3.3.1", - "version_normalized": "3.3.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "9b5daeaffce5b926cac47923798bba91059e60e2" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/9b5daeaffce5b926cac47923798bba91059e60e2", - "type": "zip", - "shasum": "", - "reference": "9b5daeaffce5b926cac47923798bba91059e60e2" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.3.1" - }, - "time": "2023-02-06T13:46:10+00:00", - "require-dev": { - "ext-json": "*", - "aws/aws-sdk-php": "^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7 || ^8", - "graylog2/gelf-php": "^1.4.2 || ^2@dev", - "guzzlehttp/guzzle": "^7.4.5", - "guzzlehttp/psr7": "^2.2", - "mongodb/mongodb": "^1.8", - "php-amqplib/php-amqplib": "~2.4 || ^3", - "phpstan/phpstan": "^1.9", - "phpstan/phpstan-deprecation-rules": "^1.0", - "phpstan/phpstan-strict-rules": "^1.4", - "phpunit/phpunit": "^9.5.26", - "predis/predis": "^1.1 || ^2", - "ruflin/elastica": "^7", - "symfony/mailer": "^5.4 || ^6", - "symfony/mime": "^5.4 || ^6" - } - }, - { - "version": "3.3.0", - "version_normalized": "3.3.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "852643b696e755f7936e80afffc6721a20f0d15c" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/852643b696e755f7936e80afffc6721a20f0d15c", - "type": "zip", - "shasum": "", - "reference": "852643b696e755f7936e80afffc6721a20f0d15c" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.3.0" - }, - "time": "2023-02-06T13:12:20+00:00" - }, - { - "version": "3.2.0", - "version_normalized": "3.2.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "305444bc6fb6c89e490f4b34fa6e979584d7fa81" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/305444bc6fb6c89e490f4b34fa6e979584d7fa81", - "type": "zip", - "shasum": "", - "reference": "305444bc6fb6c89e490f4b34fa6e979584d7fa81" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.2.0" - }, - "time": "2022-07-24T12:00:55+00:00", - "require-dev": { - "ext-json": "*", - "aws/aws-sdk-php": "^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7 || ^8", - "graylog2/gelf-php": "^1.4.2", - "guzzlehttp/guzzle": "^7.4", - "guzzlehttp/psr7": "^2.2", - "mongodb/mongodb": "^1.8", - "php-amqplib/php-amqplib": "~2.4 || ^3", - "phpstan/phpstan": "^1.4", - "phpstan/phpstan-deprecation-rules": "^1.0", - "phpstan/phpstan-strict-rules": "^1.1", - "phpunit/phpunit": "^9.5.16", - "predis/predis": "^1.1", - "ruflin/elastica": "^7", - "symfony/mailer": "^5.4 || ^6", - "symfony/mime": "^5.4 || ^6" - } - }, - { - "version": "3.1.0", - "version_normalized": "3.1.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "0c375495d40df0207e5833dca333f963b171ff43" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/0c375495d40df0207e5833dca333f963b171ff43", - "type": "zip", - "shasum": "", - "reference": "0c375495d40df0207e5833dca333f963b171ff43" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.1.0" - }, - "time": "2022-06-09T09:09:00+00:00", - "require-dev": { - "ext-json": "*", - "aws/aws-sdk-php": "^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7 || ^8", - "graylog2/gelf-php": "^1.4.2", - "guzzlehttp/guzzle": "^7.4", - "guzzlehttp/psr7": "^2.2", - "mongodb/mongodb": "^1.8", - "php-amqplib/php-amqplib": "~2.4 || ^3", - "php-console/php-console": "^3.1.3", - "phpstan/phpstan": "^1.4", - "phpstan/phpstan-deprecation-rules": "^1.0", - "phpstan/phpstan-strict-rules": "^1.1", - "phpunit/phpunit": "^9.5.16", - "predis/predis": "^1.1", - "ruflin/elastica": "^7", - "symfony/mailer": "^5.4 || ^6", - "symfony/mime": "^5.4 || ^6" - }, - "suggest": { - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "elasticsearch/elasticsearch": "Allow sending log messages to an Elasticsearch server via official client", - "php-amqplib/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongodb": "Allow sending log messages to a MongoDB server (via driver)", - "mongodb/mongodb": "Allow sending log messages to a MongoDB server (via library)", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "rollbar/rollbar": "Allow sending log messages to Rollbar", - "php-console/php-console": "Allow sending log messages to Google Chrome", - "ext-mbstring": "Allow to work properly with unicode symbols", - "ext-sockets": "Allow sending log messages to a Syslog server (via UDP driver)", - "ext-curl": "Required to send log messages using the IFTTTHandler, the LogglyHandler, the SendGridHandler, the SlackWebhookHandler or the TelegramBotHandler", - "ext-openssl": "Required to send log messages using SSL" - } - }, - { - "version": "3.0.0", - "version_normalized": "3.0.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "60ad5183b5e5d6c9d4047e9f3072d36071dcc161" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/60ad5183b5e5d6c9d4047e9f3072d36071dcc161", - "type": "zip", - "shasum": "", - "reference": "60ad5183b5e5d6c9d4047e9f3072d36071dcc161" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.0.0" - }, - "time": "2022-05-10T10:39:55+00:00" - }, - { - "version": "3.0.0-RC1", - "version_normalized": "3.0.0.0-RC1", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "a71c4e02502dd04d91f1c7d72ffccf0bd11310eb" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/a71c4e02502dd04d91f1c7d72ffccf0bd11310eb", - "type": "zip", - "shasum": "", - "reference": "a71c4e02502dd04d91f1c7d72ffccf0bd11310eb" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.0.0-RC1" - }, - "time": "2022-05-08T21:50:49+00:00" - }, - { - "version": "2.10.0", - "version_normalized": "2.10.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "5cf826f2991858b54d5c3809bee745560a1042a7" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/5cf826f2991858b54d5c3809bee745560a1042a7", - "type": "zip", - "shasum": "", - "reference": "5cf826f2991858b54d5c3809bee745560a1042a7" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.10.0" - }, - "time": "2024-11-12T12:43:37+00:00", - "extra": { - "branch-alias": { - "dev-main": "2.x-dev" - } - }, - "require": { - "php": ">=7.2", - "psr/log": "^1.0.1 || ^2.0 || ^3.0" - }, - "require-dev": { - "ext-json": "*", - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7 || ^8", - "graylog2/gelf-php": "^1.4.2 || ^2@dev", - "guzzlehttp/guzzle": "^7.4", - "guzzlehttp/psr7": "^2.2", - "mongodb/mongodb": "^1.8", - "php-amqplib/php-amqplib": "~2.4 || ^3", - "phpspec/prophecy": "^1.15", - "phpstan/phpstan": "^1.10", - "phpunit/phpunit": "^8.5.38 || ^9.6.19", - "predis/predis": "^1.1 || ^2.0", - "rollbar/rollbar": "^1.3 || ^2 || ^3", - "ruflin/elastica": "^7", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "symfony/mailer": "^5.4 || ^6", - "symfony/mime": "^5.4 || ^6" - }, - "suggest": { - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "elasticsearch/elasticsearch": "Allow sending log messages to an Elasticsearch server via official client", - "php-amqplib/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongodb": "Allow sending log messages to a MongoDB server (via driver)", - "mongodb/mongodb": "Allow sending log messages to a MongoDB server (via library)", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "rollbar/rollbar": "Allow sending log messages to Rollbar", - "ext-mbstring": "Allow to work properly with unicode symbols", - "ext-sockets": "Allow sending log messages to a Syslog server (via UDP driver)", - "ext-curl": "Required to send log messages using the IFTTTHandler, the LogglyHandler, the SendGridHandler, the SlackWebhookHandler or the TelegramBotHandler", - "ext-openssl": "Required to send log messages using SSL" - }, - "provide": { - "psr/log-implementation": "1.0.0 || 2.0.0 || 3.0.0" - } - }, - { - "version": "2.9.3", - "version_normalized": "2.9.3.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "a30bfe2e142720dfa990d0a7e573997f5d884215" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/a30bfe2e142720dfa990d0a7e573997f5d884215", - "type": "zip", - "shasum": "", - "reference": "a30bfe2e142720dfa990d0a7e573997f5d884215" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.9.3" - }, - "time": "2024-04-12T20:52:51+00:00" - }, - { - "version": "2.9.2", - "version_normalized": "2.9.2.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "437cb3628f4cf6042cc10ae97fc2b8472e48ca1f" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/437cb3628f4cf6042cc10ae97fc2b8472e48ca1f", - "type": "zip", - "shasum": "", - "reference": "437cb3628f4cf6042cc10ae97fc2b8472e48ca1f" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.9.2" - }, - "time": "2023-10-27T15:25:26+00:00", - "require-dev": { - "ext-json": "*", - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7 || ^8", - "graylog2/gelf-php": "^1.4.2 || ^2@dev", - "guzzlehttp/guzzle": "^7.4", - "guzzlehttp/psr7": "^2.2", - "mongodb/mongodb": "^1.8", - "php-amqplib/php-amqplib": "~2.4 || ^3", - "phpspec/prophecy": "^1.15", - "phpstan/phpstan": "^0.12.91", - "phpunit/phpunit": "^8.5.14", - "predis/predis": "^1.1 || ^2.0", - "rollbar/rollbar": "^1.3 || ^2 || ^3", - "ruflin/elastica": "^7", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "symfony/mailer": "^5.4 || ^6", - "symfony/mime": "^5.4 || ^6" - } - }, - { - "version": "2.9.1", - "version_normalized": "2.9.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "f259e2b15fb95494c83f52d3caad003bbf5ffaa1" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/f259e2b15fb95494c83f52d3caad003bbf5ffaa1", - "type": "zip", - "shasum": "", - "reference": "f259e2b15fb95494c83f52d3caad003bbf5ffaa1" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.9.1" - }, - "time": "2023-02-06T13:44:46+00:00" - }, - { - "version": "2.9.0", - "version_normalized": "2.9.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "e1c0ae1528ce313a450e5e1ad782765c4a8dd3cb" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/e1c0ae1528ce313a450e5e1ad782765c4a8dd3cb", - "type": "zip", - "shasum": "", - "reference": "e1c0ae1528ce313a450e5e1ad782765c4a8dd3cb" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.9.0" - }, - "time": "2023-02-05T13:07:32+00:00" - }, - { - "version": "2.8.0", - "version_normalized": "2.8.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "720488632c590286b88b80e62aa3d3d551ad4a50" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/720488632c590286b88b80e62aa3d3d551ad4a50", - "type": "zip", - "shasum": "", - "reference": "720488632c590286b88b80e62aa3d3d551ad4a50" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.8.0" - }, - "time": "2022-07-24T11:55:47+00:00", - "require-dev": { - "ext-json": "*", - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7 || ^8", - "graylog2/gelf-php": "^1.4.2", - "guzzlehttp/guzzle": "^7.4", - "guzzlehttp/psr7": "^2.2", - "mongodb/mongodb": "^1.8", - "php-amqplib/php-amqplib": "~2.4 || ^3", - "phpspec/prophecy": "^1.15", - "phpstan/phpstan": "^0.12.91", - "phpunit/phpunit": "^8.5.14", - "predis/predis": "^1.1 || ^2.0", - "rollbar/rollbar": "^1.3 || ^2 || ^3", - "ruflin/elastica": "^7", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "symfony/mailer": "^5.4 || ^6", - "symfony/mime": "^5.4 || ^6" - } - }, - { - "version": "2.7.0", - "version_normalized": "2.7.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "5579edf28aee1190a798bfa5be8bc16c563bd524" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/5579edf28aee1190a798bfa5be8bc16c563bd524", - "type": "zip", - "shasum": "", - "reference": "5579edf28aee1190a798bfa5be8bc16c563bd524" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.7.0" - }, - "time": "2022-06-09T08:59:12+00:00", - "require-dev": { - "ext-json": "*", - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7 || ^8", - "graylog2/gelf-php": "^1.4.2", - "guzzlehttp/guzzle": "^7.4", - "guzzlehttp/psr7": "^2.2", - "mongodb/mongodb": "^1.8", - "php-amqplib/php-amqplib": "~2.4 || ^3", - "php-console/php-console": "^3.1.3", - "phpspec/prophecy": "^1.15", - "phpstan/phpstan": "^0.12.91", - "phpunit/phpunit": "^8.5.14", - "predis/predis": "^1.1", - "rollbar/rollbar": "^1.3 || ^2 || ^3", - "ruflin/elastica": "^7", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "symfony/mailer": "^5.4 || ^6", - "symfony/mime": "^5.4 || ^6" - }, - "suggest": { - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "elasticsearch/elasticsearch": "Allow sending log messages to an Elasticsearch server via official client", - "php-amqplib/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongodb": "Allow sending log messages to a MongoDB server (via driver)", - "mongodb/mongodb": "Allow sending log messages to a MongoDB server (via library)", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "rollbar/rollbar": "Allow sending log messages to Rollbar", - "php-console/php-console": "Allow sending log messages to Google Chrome", - "ext-mbstring": "Allow to work properly with unicode symbols", - "ext-sockets": "Allow sending log messages to a Syslog server (via UDP driver)", - "ext-curl": "Required to send log messages using the IFTTTHandler, the LogglyHandler, the SendGridHandler, the SlackWebhookHandler or the TelegramBotHandler", - "ext-openssl": "Required to send log messages using SSL" - } - }, - { - "version": "2.6.0", - "version_normalized": "2.6.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "247918972acd74356b0a91dfaa5adcaec069b6c0" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/247918972acd74356b0a91dfaa5adcaec069b6c0", - "type": "zip", - "shasum": "", - "reference": "247918972acd74356b0a91dfaa5adcaec069b6c0" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.6.0" - }, - "time": "2022-05-10T09:36:00+00:00" - }, - { - "version": "2.5.0", - "version_normalized": "2.5.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "4192345e260f1d51b365536199744b987e160edc" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/4192345e260f1d51b365536199744b987e160edc", - "type": "zip", - "shasum": "", - "reference": "4192345e260f1d51b365536199744b987e160edc" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.5.0" - }, - "time": "2022-04-08T15:43:54+00:00", - "require-dev": { - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7", - "mongodb/mongodb": "^1.8", - "graylog2/gelf-php": "^1.4.2", - "php-amqplib/php-amqplib": "~2.4 || ^3", - "php-console/php-console": "^3.1.3", - "phpspec/prophecy": "^1.6.1", - "phpunit/phpunit": "^8.5", - "predis/predis": "^1.1", - "rollbar/rollbar": "^1.3 || ^2 || ^3", - "ruflin/elastica": ">=0.90@dev", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "phpstan/phpstan": "^0.12.91" - } - }, - { - "version": "2.4.0", - "version_normalized": "2.4.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "d7fd7450628561ba697b7097d86db72662f54aef" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/d7fd7450628561ba697b7097d86db72662f54aef", - "type": "zip", - "shasum": "", - "reference": "d7fd7450628561ba697b7097d86db72662f54aef" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.4.0" - }, - "time": "2022-03-14T12:44:37+00:00" - }, - { - "version": "2.3.5", - "version_normalized": "2.3.5.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "fd4380d6fc37626e2f799f29d91195040137eba9" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/fd4380d6fc37626e2f799f29d91195040137eba9", - "type": "zip", - "shasum": "", - "reference": "fd4380d6fc37626e2f799f29d91195040137eba9" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.3.5" - }, - "time": "2021-10-01T21:08:31+00:00", - "require-dev": { - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7", - "mongodb/mongodb": "^1.8", - "graylog2/gelf-php": "^1.4.2", - "php-amqplib/php-amqplib": "~2.4 || ^3", - "php-console/php-console": "^3.1.3", - "phpspec/prophecy": "^1.6.1", - "phpunit/phpunit": "^8.5", - "predis/predis": "^1.1", - "rollbar/rollbar": "^1.3", - "ruflin/elastica": ">=0.90@dev", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "phpstan/phpstan": "^0.12.91" - } - }, - { - "version": "2.3.4", - "version_normalized": "2.3.4.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "437e7a1c50044b92773b361af77620efb76fff59" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/437e7a1c50044b92773b361af77620efb76fff59", - "type": "zip", - "shasum": "", - "reference": "437e7a1c50044b92773b361af77620efb76fff59" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.3.4" - }, - "time": "2021-09-15T11:27:21+00:00", - "require-dev": { - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7", - "mongodb/mongodb": "^1.8", - "graylog2/gelf-php": "^1.4.2", - "php-amqplib/php-amqplib": "~2.4", - "php-console/php-console": "^3.1.3", - "phpspec/prophecy": "^1.6.1", - "phpunit/phpunit": "^8.5", - "predis/predis": "^1.1", - "rollbar/rollbar": "^1.3", - "ruflin/elastica": ">=0.90@dev", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "phpstan/phpstan": "^0.12.91" - } - }, - { - "version": "2.3.3", - "version_normalized": "2.3.3.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "3962ebfe206ac7ce6c754c79e2fee0c64bf1818d" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/3962ebfe206ac7ce6c754c79e2fee0c64bf1818d", - "type": "zip", - "shasum": "", - "reference": "3962ebfe206ac7ce6c754c79e2fee0c64bf1818d" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.3.3" - }, - "time": "2021-09-14T18:40:13+00:00", - "require": { - "php": ">=7.2", - "psr/log": "^1.0.1 || ^2.0" - }, - "require-dev": { - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7", - "mongodb/mongodb": "^1.8", - "graylog2/gelf-php": "^1.4.2", - "php-amqplib/php-amqplib": "~2.4", - "php-console/php-console": "^3.1.3", - "phpspec/prophecy": "^1.6.1", - "phpunit/phpunit": "^8.5", - "predis/predis": "^1.1", - "ruflin/elastica": ">=0.90@dev", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "phpstan/phpstan": "^0.12.91" - }, - "provide": { - "psr/log-implementation": "1.0.0 || 2.0.0" - } - }, - { - "version": "2.3.2", - "version_normalized": "2.3.2.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "71312564759a7db5b789296369c1a264efc43aad" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/71312564759a7db5b789296369c1a264efc43aad", - "type": "zip", - "shasum": "", - "reference": "71312564759a7db5b789296369c1a264efc43aad" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.3.2" - }, - "time": "2021-07-23T07:42:52+00:00", - "require": { - "php": ">=7.2", - "psr/log": "^1.0.1" - }, - "require-dev": { - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7", - "mongodb/mongodb": "^1.8", - "graylog2/gelf-php": "^1.4.2", - "php-amqplib/php-amqplib": "~2.4", - "php-console/php-console": "^3.1.3", - "phpspec/prophecy": "^1.6.1", - "phpunit/phpunit": "^8.5", - "predis/predis": "^1.1", - "rollbar/rollbar": "^1.3", - "ruflin/elastica": ">=0.90 <7.0.1", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "phpstan/phpstan": "^0.12.91" - }, - "suggest": { - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "elasticsearch/elasticsearch": "Allow sending log messages to an Elasticsearch server via official client", - "php-amqplib/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongodb": "Allow sending log messages to a MongoDB server (via driver)", - "mongodb/mongodb": "Allow sending log messages to a MongoDB server (via library)", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "rollbar/rollbar": "Allow sending log messages to Rollbar", - "php-console/php-console": "Allow sending log messages to Google Chrome", - "ext-mbstring": "Allow to work properly with unicode symbols" - }, - "provide": { - "psr/log-implementation": "1.0.0" - } - }, - { - "version": "2.3.1", - "version_normalized": "2.3.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "9738e495f288eec0b187e310b7cdbbb285777dbe" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/9738e495f288eec0b187e310b7cdbbb285777dbe", - "type": "zip", - "shasum": "", - "reference": "9738e495f288eec0b187e310b7cdbbb285777dbe" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.3.1" - }, - "time": "2021-07-14T11:56:39+00:00" - }, - { - "version": "2.3.0", - "version_normalized": "2.3.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "df991fd88693ab703aa403413d83e15f688dae33" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/df991fd88693ab703aa403413d83e15f688dae33", - "type": "zip", - "shasum": "", - "reference": "df991fd88693ab703aa403413d83e15f688dae33" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.3.0" - }, - "time": "2021-07-05T11:34:13+00:00" - }, - { - "version": "2.2.0", - "version_normalized": "2.2.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "1cb1cde8e8dd0f70cc0fe51354a59acad9302084" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/1cb1cde8e8dd0f70cc0fe51354a59acad9302084", - "type": "zip", - "shasum": "", - "reference": "1cb1cde8e8dd0f70cc0fe51354a59acad9302084" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.2.0" - }, - "time": "2020-12-14T13:15:25+00:00", - "require-dev": { - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^7", - "mongodb/mongodb": "^1.8", - "graylog2/gelf-php": "^1.4.2", - "php-amqplib/php-amqplib": "~2.4", - "php-console/php-console": "^3.1.3", - "phpspec/prophecy": "^1.6.1", - "phpunit/phpunit": "^8.5", - "predis/predis": "^1.1", - "rollbar/rollbar": "^1.3", - "ruflin/elastica": ">=0.90 <7.0.1", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "phpstan/phpstan": "^0.12.59" - } - }, - { - "homepage": "http://github.com/Seldaek/monolog", - "version": "2.1.1", - "version_normalized": "2.1.1.0", - "authors": [ - { - "name": "Jordi Boggiano", - "email": "j.boggiano@seld.be", - "homepage": "http://seld.be" - } - ], - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "f9eee5cec93dfb313a38b6b288741e84e53f02d5" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/f9eee5cec93dfb313a38b6b288741e84e53f02d5", - "type": "zip", - "shasum": "", - "reference": "f9eee5cec93dfb313a38b6b288741e84e53f02d5" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.1.1" - }, - "time": "2020-07-23T08:41:23+00:00", - "extra": { - "branch-alias": { - "dev-master": "2.x-dev" - } - }, - "require-dev": { - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^6.0", - "graylog2/gelf-php": "^1.4.2", - "php-amqplib/php-amqplib": "~2.4", - "php-console/php-console": "^3.1.3", - "php-parallel-lint/php-parallel-lint": "^1.0", - "phpspec/prophecy": "^1.6.1", - "phpunit/phpunit": "^8.5", - "predis/predis": "^1.1", - "rollbar/rollbar": "^1.3", - "ruflin/elastica": ">=0.90 <3.0", - "swiftmailer/swiftmailer": "^5.3|^6.0" - } - }, - { - "version": "2.1.0", - "version_normalized": "2.1.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "38914429aac460e8e4616c8cb486ecb40ec90bb1" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/38914429aac460e8e4616c8cb486ecb40ec90bb1", - "type": "zip", - "shasum": "", - "reference": "38914429aac460e8e4616c8cb486ecb40ec90bb1" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.1.0" - }, - "time": "2020-05-22T08:12:19+00:00" - }, - { - "version": "2.0.2", - "version_normalized": "2.0.2.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "c861fcba2ca29404dc9e617eedd9eff4616986b8" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/c861fcba2ca29404dc9e617eedd9eff4616986b8", - "type": "zip", - "shasum": "", - "reference": "c861fcba2ca29404dc9e617eedd9eff4616986b8" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.0.2" - }, - "time": "2019-12-20T14:22:59+00:00", - "require": { - "php": "^7.2", - "psr/log": "^1.0.1" - }, - "require-dev": { - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^6.0", - "graylog2/gelf-php": "^1.4.2", - "jakub-onderka/php-parallel-lint": "^0.9", - "php-amqplib/php-amqplib": "~2.4", - "php-console/php-console": "^3.1.3", - "phpspec/prophecy": "^1.6.1", - "phpunit/phpunit": "^8.3", - "predis/predis": "^1.1", - "rollbar/rollbar": "^1.3", - "ruflin/elastica": ">=0.90 <3.0", - "swiftmailer/swiftmailer": "^5.3|^6.0" - } - }, - { - "version": "2.0.1", - "version_normalized": "2.0.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "f9d56fd2f5533322caccdfcddbb56aedd622ef1c" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/f9d56fd2f5533322caccdfcddbb56aedd622ef1c", - "type": "zip", - "shasum": "", - "reference": "f9d56fd2f5533322caccdfcddbb56aedd622ef1c" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.0.1" - }, - "time": "2019-11-13T10:27:43+00:00" - }, - { - "version": "2.0.0", - "version_normalized": "2.0.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "68545165e19249013afd1d6f7485aecff07a2d22" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/68545165e19249013afd1d6f7485aecff07a2d22", - "type": "zip", - "shasum": "", - "reference": "68545165e19249013afd1d6f7485aecff07a2d22" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.0.0" - }, - "time": "2019-08-30T09:56:44+00:00" - }, - { - "version": "2.0.0-beta2", - "version_normalized": "2.0.0.0-beta2", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "bf486002a08ca7cb156540e1a38c7be70fa8ed59" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/bf486002a08ca7cb156540e1a38c7be70fa8ed59", - "type": "zip", - "shasum": "", - "reference": "bf486002a08ca7cb156540e1a38c7be70fa8ed59" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.0.0-beta2" - }, - "time": "2019-07-06T13:17:41+00:00", - "require-dev": { - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "doctrine/couchdb": "~1.0@dev", - "elasticsearch/elasticsearch": "^6.0", - "graylog2/gelf-php": "^1.4.2", - "jakub-onderka/php-parallel-lint": "^0.9", - "php-amqplib/php-amqplib": "~2.4", - "php-console/php-console": "^3.1.3", - "phpspec/prophecy": "^1.6.1", - "phpunit/phpunit": "^7.5", - "predis/predis": "^1.1", - "rollbar/rollbar": "^1.3", - "ruflin/elastica": ">=0.90 <3.0", - "swiftmailer/swiftmailer": "^5.3|^6.0" - }, - "suggest": { - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "elasticsearch/elasticsearch": "Allow sending log messages to an Elasticsearch server via official client", - "php-amqplib/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongodb": "Allow sending log messages to a MongoDB server (via driver)", - "mongodb/mongodb": "Allow sending log messages to a MongoDB server (via library)", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "rollbar/rollbar": "Allow sending log messages to Rollbar", - "php-console/php-console": "Allow sending log messages to Google Chrome" - } - }, - { - "version": "2.0.0-beta1", - "version_normalized": "2.0.0.0-beta1", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "0ad73a526f4b5e67312e94fb7f60c1bdefc284b9" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/0ad73a526f4b5e67312e94fb7f60c1bdefc284b9", - "type": "zip", - "shasum": "", - "reference": "0ad73a526f4b5e67312e94fb7f60c1bdefc284b9" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/2.0.0-beta1" - }, - "time": "2018-12-08T17:16:32+00:00", - "require": { - "php": "^7.1", - "psr/log": "^1.0.1" - }, - "require-dev": { - "phpunit/phpunit": "^7.3", - "graylog2/gelf-php": "^1.4.2", - "sentry/sentry": "^1.9", - "ruflin/elastica": ">=0.90 <3.0", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "php-amqplib/php-amqplib": "~2.4", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "php-console/php-console": "^3.1.3", - "jakub-onderka/php-parallel-lint": "^0.9", - "predis/predis": "^1.1", - "phpspec/prophecy": "^1.6.1", - "elasticsearch/elasticsearch": "^6.0", - "rollbar/rollbar": "^1.3" - }, - "suggest": { - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "sentry/sentry": "Allow sending log messages to a Sentry server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "elasticsearch/elasticsearch": "Allow sending log messages to an Elasticsearch server via official client", - "php-amqplib/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongodb": "Allow sending log messages to a MongoDB server (via driver)", - "mongodb/mongodb": "Allow sending log messages to a MongoDB server (via library)", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "rollbar/rollbar": "Allow sending log messages to Rollbar", - "php-console/php-console": "Allow sending log messages to Google Chrome" - } - }, - { - "version": "1.27.1", - "version_normalized": "1.27.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "904713c5929655dc9b97288b69cfeedad610c9a1" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/904713c5929655dc9b97288b69cfeedad610c9a1", - "type": "zip", - "shasum": "", - "reference": "904713c5929655dc9b97288b69cfeedad610c9a1" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.27.1" - }, - "time": "2022-06-09T08:53:42+00:00", - "require": { - "php": ">=5.3.0", - "psr/log": "~1.0" - }, - "require-dev": { - "phpunit/phpunit": "~4.5", - "graylog2/gelf-php": "~1.0", - "sentry/sentry": "^0.13", - "ruflin/elastica": ">=0.90 <3.0", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "php-amqplib/php-amqplib": "~2.4", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "php-console/php-console": "^3.1.3", - "phpstan/phpstan": "^0.12.59" - }, - "suggest": { - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "sentry/sentry": "Allow sending log messages to a Sentry server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "php-amqplib/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongo": "Allow sending log messages to a MongoDB server", - "mongodb/mongodb": "Allow sending log messages to a MongoDB server via PHP Driver", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "rollbar/rollbar": "Allow sending log messages to Rollbar", - "php-console/php-console": "Allow sending log messages to Google Chrome" - }, - "extra": "__unset" - }, - { - "version": "1.27.0", - "version_normalized": "1.27.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "52ebd235c1f7e0d5e1b16464b695a28335f8e44a" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/52ebd235c1f7e0d5e1b16464b695a28335f8e44a", - "type": "zip", - "shasum": "", - "reference": "52ebd235c1f7e0d5e1b16464b695a28335f8e44a" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.27.0" - }, - "time": "2022-03-13T20:29:46+00:00" - }, - { - "version": "1.26.1", - "version_normalized": "1.26.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "c6b00f05152ae2c9b04a448f99c7590beb6042f5" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/c6b00f05152ae2c9b04a448f99c7590beb6042f5", - "type": "zip", - "shasum": "", - "reference": "c6b00f05152ae2c9b04a448f99c7590beb6042f5" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.26.1" - }, - "time": "2021-05-28T08:32:12+00:00" - }, - { - "version": "1.26.0", - "version_normalized": "1.26.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "2209ddd84e7ef1256b7af205d0717fb62cfc9c33" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/2209ddd84e7ef1256b7af205d0717fb62cfc9c33", - "type": "zip", - "shasum": "", - "reference": "2209ddd84e7ef1256b7af205d0717fb62cfc9c33" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.26.0" - }, - "time": "2020-12-14T12:56:38+00:00" - }, - { - "version": "1.25.5", - "version_normalized": "1.25.5.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "1817faadd1846cd08be9a49e905dc68823bc38c0" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/1817faadd1846cd08be9a49e905dc68823bc38c0", - "type": "zip", - "shasum": "", - "reference": "1817faadd1846cd08be9a49e905dc68823bc38c0" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.25.5" - }, - "time": "2020-07-23T08:35:51+00:00", - "extra": { - "branch-alias": { - "dev-master": "2.0.x-dev" - } - }, - "require-dev": { - "phpunit/phpunit": "~4.5", - "graylog2/gelf-php": "~1.0", - "sentry/sentry": "^0.13", - "ruflin/elastica": ">=0.90 <3.0", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "php-amqplib/php-amqplib": "~2.4", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "php-console/php-console": "^3.1.3", - "php-parallel-lint/php-parallel-lint": "^1.0" - } - }, - { - "version": "1.25.4", - "version_normalized": "1.25.4.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "3022efff205e2448b560c833c6fbbf91c3139168" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/3022efff205e2448b560c833c6fbbf91c3139168", - "type": "zip", - "shasum": "", - "reference": "3022efff205e2448b560c833c6fbbf91c3139168" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.25.4" - }, - "time": "2020-05-22T07:31:27+00:00" - }, - { - "version": "1.25.3", - "version_normalized": "1.25.3.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "fa82921994db851a8becaf3787a9e73c5976b6f1" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/fa82921994db851a8becaf3787a9e73c5976b6f1", - "type": "zip", - "shasum": "", - "reference": "fa82921994db851a8becaf3787a9e73c5976b6f1" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.25.3" - }, - "time": "2019-12-20T14:15:16+00:00", - "require-dev": { - "phpunit/phpunit": "~4.5", - "graylog2/gelf-php": "~1.0", - "sentry/sentry": "^0.13", - "ruflin/elastica": ">=0.90 <3.0", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "php-amqplib/php-amqplib": "~2.4", - "swiftmailer/swiftmailer": "^5.3|^6.0", - "php-console/php-console": "^3.1.3", - "phpunit/phpunit-mock-objects": "2.3.0", - "jakub-onderka/php-parallel-lint": "0.9" - } - }, - { - "version": "1.25.2", - "version_normalized": "1.25.2.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "d5e2fb341cb44f7e2ab639d12a1e5901091ec287" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/d5e2fb341cb44f7e2ab639d12a1e5901091ec287", - "type": "zip", - "shasum": "", - "reference": "d5e2fb341cb44f7e2ab639d12a1e5901091ec287" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.25.2" - }, - "time": "2019-11-13T10:00:05+00:00" - }, - { - "version": "1.25.1", - "version_normalized": "1.25.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "70e65a5470a42cfec1a7da00d30edb6e617e8dcf" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/70e65a5470a42cfec1a7da00d30edb6e617e8dcf", - "type": "zip", - "shasum": "", - "reference": "70e65a5470a42cfec1a7da00d30edb6e617e8dcf" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.25.1" - }, - "time": "2019-09-06T13:49:17+00:00" - }, - { - "version": "1.25.0", - "version_normalized": "1.25.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "c5dcc05defbaf8780c728c1ea31b1a0704d44f56" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/c5dcc05defbaf8780c728c1ea31b1a0704d44f56", - "type": "zip", - "shasum": "", - "reference": "c5dcc05defbaf8780c728c1ea31b1a0704d44f56" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.25.0" - }, - "time": "2019-09-06T12:21:24+00:00" - }, - { - "version": "1.24.0", - "version_normalized": "1.24.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "bfc9ebb28f97e7a24c45bdc3f0ff482e47bb0266" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/bfc9ebb28f97e7a24c45bdc3f0ff482e47bb0266", - "type": "zip", - "shasum": "", - "reference": "bfc9ebb28f97e7a24c45bdc3f0ff482e47bb0266" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.24.0" - }, - "time": "2018-11-05T09:00:11+00:00" - }, - { - "version": "1.23.0", - "version_normalized": "1.23.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "fd8c787753b3a2ad11bc60c063cff1358a32a3b4" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/fd8c787753b3a2ad11bc60c063cff1358a32a3b4", - "type": "zip", - "shasum": "", - "reference": "fd8c787753b3a2ad11bc60c063cff1358a32a3b4" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.23.0" - }, - "time": "2017-06-19T01:22:40+00:00" - }, - { - "version": "1.22.1", - "version_normalized": "1.22.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "1e044bc4b34e91743943479f1be7a1d5eb93add0" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/1e044bc4b34e91743943479f1be7a1d5eb93add0", - "type": "zip", - "shasum": "", - "reference": "1e044bc4b34e91743943479f1be7a1d5eb93add0" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.22.1" - }, - "time": "2017-03-13T07:08:03+00:00", - "require-dev": { - "phpunit/phpunit": "~4.5", - "graylog2/gelf-php": "~1.0", - "sentry/sentry": "^0.13", - "ruflin/elastica": ">=0.90 <3.0", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "^2.4.9 || ^3.0", - "php-amqplib/php-amqplib": "~2.4", - "swiftmailer/swiftmailer": "~5.3", - "php-console/php-console": "^3.1.3", - "phpunit/phpunit-mock-objects": "2.3.0", - "jakub-onderka/php-parallel-lint": "0.9" - } - }, - { - "version": "1.22.0", - "version_normalized": "1.22.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "bad29cb8d18ab0315e6c477751418a82c850d558" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/bad29cb8d18ab0315e6c477751418a82c850d558", - "type": "zip", - "shasum": "", - "reference": "bad29cb8d18ab0315e6c477751418a82c850d558" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.22.0" - }, - "time": "2016-11-26T00:15:39+00:00" - }, - { - "version": "1.21.0", - "version_normalized": "1.21.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "f42fbdfd53e306bda545845e4dbfd3e72edb4952" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/f42fbdfd53e306bda545845e4dbfd3e72edb4952", - "type": "zip", - "shasum": "", - "reference": "f42fbdfd53e306bda545845e4dbfd3e72edb4952" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.21.0" - }, - "time": "2016-07-29T03:23:52+00:00", - "require-dev": { - "phpunit/phpunit": "~4.5", - "graylog2/gelf-php": "~1.0", - "sentry/sentry": "^0.13", - "ruflin/elastica": ">=0.90 <3.0", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "^2.4.9", - "php-amqplib/php-amqplib": "~2.4", - "swiftmailer/swiftmailer": "~5.3", - "php-console/php-console": "^3.1.3", - "phpunit/phpunit-mock-objects": "2.3.0", - "jakub-onderka/php-parallel-lint": "0.9" - } - }, - { - "version": "1.20.0", - "version_normalized": "1.20.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "55841909e2bcde01b5318c35f2b74f8ecc86e037" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/55841909e2bcde01b5318c35f2b74f8ecc86e037", - "type": "zip", - "shasum": "", - "reference": "55841909e2bcde01b5318c35f2b74f8ecc86e037" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.20.0" - }, - "time": "2016-07-02T14:02:10+00:00" - }, - { - "version": "1.19.0", - "version_normalized": "1.19.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "5f56ed5212dc509c8dc8caeba2715732abb32dbf" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/5f56ed5212dc509c8dc8caeba2715732abb32dbf", - "type": "zip", - "shasum": "", - "reference": "5f56ed5212dc509c8dc8caeba2715732abb32dbf" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.19.0" - }, - "time": "2016-04-12T18:29:35+00:00", - "require-dev": { - "phpunit/phpunit": "~4.5", - "graylog2/gelf-php": "~1.0", - "raven/raven": "^0.13", - "ruflin/elastica": ">=0.90 <3.0", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "^2.4.9", - "php-amqplib/php-amqplib": "~2.4", - "swiftmailer/swiftmailer": "~5.3", - "php-console/php-console": "^3.1.3", - "phpunit/phpunit-mock-objects": "2.3.0", - "jakub-onderka/php-parallel-lint": "0.9" - }, - "suggest": { - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "raven/raven": "Allow sending log messages to a Sentry server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "php-amqplib/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongo": "Allow sending log messages to a MongoDB server", - "mongodb/mongodb": "Allow sending log messages to a MongoDB server via PHP Driver", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "rollbar/rollbar": "Allow sending log messages to Rollbar", - "php-console/php-console": "Allow sending log messages to Google Chrome" - } - }, - { - "version": "1.18.2", - "version_normalized": "1.18.2.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "064b38c16790249488e7a8b987acf1c9d7383c09" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/064b38c16790249488e7a8b987acf1c9d7383c09", - "type": "zip", - "shasum": "", - "reference": "064b38c16790249488e7a8b987acf1c9d7383c09" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.18.2" - }, - "time": "2016-04-02T13:12:58+00:00" - }, - { - "version": "1.18.1", - "version_normalized": "1.18.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "a5f2734e8c16f3aa21b3da09715d10e15b4d2d45" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/a5f2734e8c16f3aa21b3da09715d10e15b4d2d45", - "type": "zip", - "shasum": "", - "reference": "a5f2734e8c16f3aa21b3da09715d10e15b4d2d45" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.18.1" - }, - "time": "2016-03-13T16:08:35+00:00", - "require-dev": { - "phpunit/phpunit": "~4.5", - "graylog2/gelf-php": "~1.0", - "raven/raven": "^0.13", - "ruflin/elastica": ">=0.90 <3.0", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "^2.4.9", - "videlalvaro/php-amqplib": "~2.4", - "swiftmailer/swiftmailer": "~5.3", - "php-console/php-console": "^3.1.3", - "phpunit/phpunit-mock-objects": "2.3.0", - "jakub-onderka/php-parallel-lint": "0.9" - }, - "suggest": { - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "raven/raven": "Allow sending log messages to a Sentry server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "videlalvaro/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongo": "Allow sending log messages to a MongoDB server", - "mongodb/mongodb": "Allow sending log messages to a MongoDB server via PHP Driver", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "rollbar/rollbar": "Allow sending log messages to Rollbar", - "php-console/php-console": "Allow sending log messages to Google Chrome" - } - }, - { - "version": "1.18.0", - "version_normalized": "1.18.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "e19b764b5c855580e8ffa7e615f72c10fd2f99cc" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/e19b764b5c855580e8ffa7e615f72c10fd2f99cc", - "type": "zip", - "shasum": "", - "reference": "e19b764b5c855580e8ffa7e615f72c10fd2f99cc" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.18.0" - }, - "time": "2016-03-01T18:00:40+00:00" - }, - { - "version": "1.17.2", - "version_normalized": "1.17.2.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "bee7f0dc9c3e0b69a6039697533dca1e845c8c24" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/bee7f0dc9c3e0b69a6039697533dca1e845c8c24", - "type": "zip", - "shasum": "", - "reference": "bee7f0dc9c3e0b69a6039697533dca1e845c8c24" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.17.2" - }, - "time": "2015-10-14T12:51:02+00:00", - "extra": { - "branch-alias": { - "dev-master": "1.16.x-dev" - } - }, - "suggest": { - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "raven/raven": "Allow sending log messages to a Sentry server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "videlalvaro/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongo": "Allow sending log messages to a MongoDB server", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "rollbar/rollbar": "Allow sending log messages to Rollbar", - "php-console/php-console": "Allow sending log messages to Google Chrome" - } - }, - { - "version": "1.17.1", - "version_normalized": "1.17.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "0524c87587ab85bc4c2d6f5b41253ccb930a5422" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/0524c87587ab85bc4c2d6f5b41253ccb930a5422", - "type": "zip", - "shasum": "", - "reference": "0524c87587ab85bc4c2d6f5b41253ccb930a5422" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.17.1" - }, - "time": "2015-08-31T09:17:37+00:00", - "require-dev": { - "phpunit/phpunit": "~4.5", - "graylog2/gelf-php": "~1.0", - "raven/raven": "~0.11", - "ruflin/elastica": ">=0.90 <3.0", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "^2.4.9", - "videlalvaro/php-amqplib": "~2.4", - "swiftmailer/swiftmailer": "~5.3", - "php-console/php-console": "^3.1.3", - "phpunit/phpunit-mock-objects": "2.3.0" - } - }, - { - "version": "1.17.0", - "version_normalized": "1.17.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "877ae631713cc961952df713ae785735b90df682" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/877ae631713cc961952df713ae785735b90df682", - "type": "zip", - "shasum": "", - "reference": "877ae631713cc961952df713ae785735b90df682" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.17.0" - }, - "time": "2015-08-30T11:40:25+00:00" - }, - { - "version": "1.16.0", - "version_normalized": "1.16.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "c0c0b4bee3aabce7182876b0d912ef2595563db7" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/c0c0b4bee3aabce7182876b0d912ef2595563db7", - "type": "zip", - "shasum": "", - "reference": "c0c0b4bee3aabce7182876b0d912ef2595563db7" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.16.0" - }, - "time": "2015-08-09T17:44:44+00:00", - "require-dev": { - "phpunit/phpunit": "~4.5", - "graylog2/gelf-php": "~1.0", - "raven/raven": "~0.8", - "ruflin/elastica": ">=0.90 <3.0", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "^2.4.9", - "videlalvaro/php-amqplib": "~2.4", - "swiftmailer/swiftmailer": "~5.3", - "php-console/php-console": "^3.1.3", - "phpunit/phpunit-mock-objects": "2.3.0" - } - }, - { - "version": "1.15.0", - "version_normalized": "1.15.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "dc5150cc608f2334c72c3b6a553ec9668a4156b0" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/dc5150cc608f2334c72c3b6a553ec9668a4156b0", - "type": "zip", - "shasum": "", - "reference": "dc5150cc608f2334c72c3b6a553ec9668a4156b0" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.15.0" - }, - "time": "2015-07-12T13:54:09+00:00", - "extra": { - "branch-alias": { - "dev-master": "1.15.x-dev" - } - } - }, - { - "version": "1.14.0", - "version_normalized": "1.14.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "b287fbbe1ca27847064beff2bad7fb6920bf08cc" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/b287fbbe1ca27847064beff2bad7fb6920bf08cc", - "type": "zip", - "shasum": "", - "reference": "b287fbbe1ca27847064beff2bad7fb6920bf08cc" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.14.0" - }, - "time": "2015-06-19T13:29:54+00:00", - "extra": { - "branch-alias": { - "dev-master": "1.14.x-dev" - } - } - }, - { - "version": "1.13.1", - "version_normalized": "1.13.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "c31a2c4e8db5da8b46c74cf275d7f109c0f249ac" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/c31a2c4e8db5da8b46c74cf275d7f109c0f249ac", - "type": "zip", - "shasum": "", - "reference": "c31a2c4e8db5da8b46c74cf275d7f109c0f249ac" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.13.1" - }, - "time": "2015-03-09T09:58:04+00:00", - "extra": { - "branch-alias": { - "dev-master": "1.13.x-dev" - } - }, - "require-dev": { - "phpunit/phpunit": "~4.0", - "graylog2/gelf-php": "~1.0", - "raven/raven": "~0.5", - "ruflin/elastica": "0.90.*", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "~2.4, >2.4.8", - "videlalvaro/php-amqplib": "~2.4", - "swiftmailer/swiftmailer": "~5.3" - }, - "suggest": { - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "raven/raven": "Allow sending log messages to a Sentry server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "videlalvaro/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongo": "Allow sending log messages to a MongoDB server", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "rollbar/rollbar": "Allow sending log messages to Rollbar" - } - }, - { - "version": "1.13.0", - "version_normalized": "1.13.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "c41c218e239b50446fd883acb1ecfd4b770caeae" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/c41c218e239b50446fd883acb1ecfd4b770caeae", - "type": "zip", - "shasum": "", - "reference": "c41c218e239b50446fd883acb1ecfd4b770caeae" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.13.0" - }, - "time": "2015-03-05T01:12:12+00:00" - }, - { - "version": "1.12.0", - "version_normalized": "1.12.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "1fbe8c2641f2b163addf49cc5e18f144bec6b19f" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/1fbe8c2641f2b163addf49cc5e18f144bec6b19f", - "type": "zip", - "shasum": "", - "reference": "1fbe8c2641f2b163addf49cc5e18f144bec6b19f" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.12.0" - }, - "time": "2014-12-29T21:29:35+00:00", - "extra": { - "branch-alias": { - "dev-master": "1.12.x-dev" - } - }, - "require-dev": { - "phpunit/phpunit": "~4.0", - "graylog2/gelf-php": "~1.0", - "raven/raven": "~0.5", - "ruflin/elastica": "0.90.*", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "~2.4, >2.4.8", - "videlalvaro/php-amqplib": "~2.4" - } - }, - { - "version": "1.11.0", - "version_normalized": "1.11.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "ec3961874c43840e96da3a8a1ed20d8c73d7e5aa" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/ec3961874c43840e96da3a8a1ed20d8c73d7e5aa", - "type": "zip", - "shasum": "", - "reference": "ec3961874c43840e96da3a8a1ed20d8c73d7e5aa" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.11.0" - }, - "time": "2014-09-30T13:30:58+00:00", - "extra": { - "branch-alias": { - "dev-master": "1.11.x-dev" - } - }, - "require-dev": { - "phpunit/phpunit": "~3.7.0", - "graylog2/gelf-php": "~1.0", - "raven/raven": "~0.5", - "ruflin/elastica": "0.90.*", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "~2.4, >2.4.8", - "videlalvaro/php-amqplib": "~2.4" - } - }, - { - "version": "1.10.0", - "version_normalized": "1.10.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "25b16e801979098cb2f120e697bfce454b18bf23" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/25b16e801979098cb2f120e697bfce454b18bf23", - "type": "zip", - "shasum": "", - "reference": "25b16e801979098cb2f120e697bfce454b18bf23" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.10.0" - }, - "time": "2014-06-04T16:30:04+00:00", - "extra": { - "branch-alias": { - "dev-master": "1.10.x-dev" - } - }, - "require-dev": { - "phpunit/phpunit": "~3.7.0", - "graylog2/gelf-php": "~1.0", - "raven/raven": "~0.5", - "ruflin/elastica": "0.90.*", - "doctrine/couchdb": "~1.0@dev", - "aws/aws-sdk-php": "~2.4, >2.4.8" - }, - "suggest": { - "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", - "raven/raven": "Allow sending log messages to a Sentry server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongo": "Allow sending log messages to a MongoDB server", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", - "rollbar/rollbar": "Allow sending log messages to Rollbar" - }, - "provide": "__unset" - }, - { - "version": "1.9.1", - "version_normalized": "1.9.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "65026b610f8c19e61d7242f600530677b0466aac" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/65026b610f8c19e61d7242f600530677b0466aac", - "type": "zip", - "shasum": "", - "reference": "65026b610f8c19e61d7242f600530677b0466aac" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.9.1" - }, - "time": "2014-04-24T13:29:03+00:00", - "extra": { - "branch-alias": { - "dev-master": "1.9.x-dev" - } - } - }, - { - "version": "1.9.0", - "version_normalized": "1.9.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "1afc39690e7414412face1f8cbf67b73db34485c" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/1afc39690e7414412face1f8cbf67b73db34485c", - "type": "zip", - "shasum": "", - "reference": "1afc39690e7414412face1f8cbf67b73db34485c" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.9.0" - }, - "time": "2014-04-20T16:41:26+00:00" - }, - { - "version": "1.8.0", - "version_normalized": "1.8.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "392ef35fd470638e08d0160d6b1cbab63cb23174" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/392ef35fd470638e08d0160d6b1cbab63cb23174", - "type": "zip", - "shasum": "", - "reference": "392ef35fd470638e08d0160d6b1cbab63cb23174" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.8.0" - }, - "time": "2014-03-23T19:50:26+00:00", - "extra": { - "branch-alias": { - "dev-master": "1.8.x-dev" - } - } - }, - { - "version": "1.7.0", - "version_normalized": "1.7.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "6225b22de9dcf36546be3a0b2fa8e3d986153f57" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/6225b22de9dcf36546be3a0b2fa8e3d986153f57", - "type": "zip", - "shasum": "", - "reference": "6225b22de9dcf36546be3a0b2fa8e3d986153f57" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.7.0" - }, - "time": "2013-11-14T19:48:31+00:00", - "autoload": { - "psr-0": { - "Monolog": "src/" - } - }, - "extra": { - "branch-alias": { - "dev-master": "1.7.x-dev" - } - }, - "require-dev": { - "phpunit/phpunit": "~3.7.0", - "mlehner/gelf-php": "1.0.*", - "raven/raven": "0.5.*", - "ruflin/elastica": "0.90.*", - "doctrine/couchdb": "dev-master", - "aws/aws-sdk-php": "~2.4.8" - }, - "suggest": { - "mlehner/gelf-php": "Allow sending log messages to a GrayLog2 server", - "raven/raven": "Allow sending log messages to a Sentry server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ruflin/elastica": "Allow sending log messages to an Elastic Search server", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongo": "Allow sending log messages to a MongoDB server", - "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB" - } - }, - { - "version": "1.6.0", - "version_normalized": "1.6.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "f72392d0e6eb855118f5a84e89ac2d257c704abd" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/f72392d0e6eb855118f5a84e89ac2d257c704abd", - "type": "zip", - "shasum": "", - "reference": "f72392d0e6eb855118f5a84e89ac2d257c704abd" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.6.0" - }, - "time": "2013-07-28T22:38:30+00:00", - "extra": { - "branch-alias": { - "dev-master": "1.6.x-dev" - } - }, - "require-dev": { - "mlehner/gelf-php": "1.0.*", - "raven/raven": "0.5.*", - "doctrine/couchdb": "dev-master" - }, - "suggest": { - "mlehner/gelf-php": "Allow sending log messages to a GrayLog2 server", - "raven/raven": "Allow sending log messages to a Sentry server", - "doctrine/couchdb": "Allow sending log messages to a CouchDB server", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongo": "Allow sending log messages to a MongoDB server" - } - }, - { - "version": "1.5.0", - "version_normalized": "1.5.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "583618d5cd2115a52101694aca87afb182b3e567" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/583618d5cd2115a52101694aca87afb182b3e567", - "type": "zip", - "shasum": "", - "reference": "583618d5cd2115a52101694aca87afb182b3e567" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.5.0" - }, - "time": "2013-04-23T10:09:48+00:00", - "extra": { - "branch-alias": { - "dev-master": "1.4.x-dev" - } - }, - "require-dev": { - "mlehner/gelf-php": "1.0.*", - "raven/raven": "0.3.*", - "doctrine/couchdb": "dev-master" - } - }, - { - "version": "1.4.1", - "version_normalized": "1.4.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "3295de82be06b3bbcd336983ddf8c50724430180" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/3295de82be06b3bbcd336983ddf8c50724430180", - "type": "zip", - "shasum": "", - "reference": "3295de82be06b3bbcd336983ddf8c50724430180" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.4.1" - }, - "time": "2013-04-01T10:04:58+00:00" - }, - { - "version": "1.4.0", - "version_normalized": "1.4.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "32fe28af60b4da9a5b0cef024138afacc0c01eeb" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/32fe28af60b4da9a5b0cef024138afacc0c01eeb", - "type": "zip", - "shasum": "", - "reference": "32fe28af60b4da9a5b0cef024138afacc0c01eeb" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.4.0" - }, - "time": "2013-02-13T18:06:51+00:00" - }, - { - "version": "1.3.1", - "version_normalized": "1.3.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "47eb599b4aad36b66e818ed72ebf939e2fb311be" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/47eb599b4aad36b66e818ed72ebf939e2fb311be", - "type": "zip", - "shasum": "", - "reference": "47eb599b4aad36b66e818ed72ebf939e2fb311be" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.3.1" - }, - "time": "2013-01-11T10:23:20+00:00", - "extra": { - "branch-alias": { - "dev-master": "1.3.x-dev" - } - } - }, - { - "version": "1.3.0", - "version_normalized": "1.3.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "25a97abf904120a386c546be11c3b58f1f9e6f37" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/25a97abf904120a386c546be11c3b58f1f9e6f37", - "type": "zip", - "shasum": "", - "reference": "25a97abf904120a386c546be11c3b58f1f9e6f37" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.3.0" - }, - "time": "2013-01-07T20:26:46+00:00" - }, - { - "description": "Logging for PHP 5.3", - "keywords": [ - "log", - "logging" - ], - "version": "1.2.1", - "version_normalized": "1.2.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "d16496318c3e08e3bccfc3866e104e49cf25488a" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/d16496318c3e08e3bccfc3866e104e49cf25488a", - "type": "zip", - "shasum": "", - "reference": "d16496318c3e08e3bccfc3866e104e49cf25488a" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.2.1" - }, - "time": "2012-08-29T11:53:20+00:00", - "require": { - "php": ">=5.3.0" - }, - "require-dev": { - "mlehner/gelf-php": "1.0.*" - }, - "suggest": { - "mlehner/gelf-php": "Allow sending log messages to a GrayLog2 server", - "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", - "ext-mongo": "Allow sending log messages to a MongoDB server" - } - }, - { - "version": "1.2.0", - "version_normalized": "1.2.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "7940ae31ce4687d875d2bb5aa277bb3802203fe1" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/7940ae31ce4687d875d2bb5aa277bb3802203fe1", - "type": "zip", - "shasum": "", - "reference": "7940ae31ce4687d875d2bb5aa277bb3802203fe1" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.2.0" - }, - "time": "2012-08-18T17:27:13+00:00", - "extra": "__unset" - }, - { - "version": "1.1.0", - "version_normalized": "1.1.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "abc80e0db8ad31c03b373977fc997e980800f9c2" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/abc80e0db8ad31c03b373977fc997e980800f9c2", - "type": "zip", - "shasum": "", - "reference": "abc80e0db8ad31c03b373977fc997e980800f9c2" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.1.0" - }, - "time": "2012-04-23T16:27:40+00:00", - "suggest": { - "mlehner/gelf-php": "Allow sending log messages to a GrayLog2 server" - } - }, - { - "version": "1.0.2", - "version_normalized": "1.0.2.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "b704c49a3051536f67f2d39f13568f74615b9922" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/b704c49a3051536f67f2d39f13568f74615b9922", - "type": "zip", - "shasum": "", - "reference": "b704c49a3051536f67f2d39f13568f74615b9922" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.0.2" - }, - "time": "2011-10-24T09:39:02+00:00", - "require-dev": "__unset", - "suggest": "__unset" - }, - { - "version": "1.0.1", - "version_normalized": "1.0.1.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "303b8a83c87d5c6d749926cf02620465a5dcd0f2" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/303b8a83c87d5c6d749926cf02620465a5dcd0f2", - "type": "zip", - "shasum": "", - "reference": "303b8a83c87d5c6d749926cf02620465a5dcd0f2" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.0.1" - }, - "time": "2011-08-25T20:42:58+00:00", - "autoload": "__unset" - }, - { - "version": "1.0.0", - "version_normalized": "1.0.0.0", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "433b98d4218c181bae01865901aac045585e8a1a" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/433b98d4218c181bae01865901aac045585e8a1a", - "type": "zip", - "shasum": "", - "reference": "433b98d4218c181bae01865901aac045585e8a1a" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.0.0" - }, - "time": "2011-07-07T16:21:02+00:00" - }, - { - "version": "1.0.0-RC1", - "version_normalized": "1.0.0.0-RC1", - "source": { - "url": "https://github.com/Seldaek/monolog.git", - "type": "git", - "reference": "5e651a82b4b03d267da6084720ada0cd398c8d16" - }, - "dist": { - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/5e651a82b4b03d267da6084720ada0cd398c8d16", - "type": "zip", - "shasum": "", - "reference": "5e651a82b4b03d267da6084720ada0cd398c8d16" - }, - "support": { - "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/1.0.0-RC1" - }, - "time": "2011-07-01T19:29:40+00:00" - } - ] - }, - "security-advisories": [ - { - "advisoryId": "PKSA-dmw8-jd8k-q3c6", - "affectedVersions": ">=1.8.0,<1.12.0" - } - ] -} \ No newline at end of file +[ + "pkg:composer/monolog/monolog@3.9.0", + "pkg:composer/monolog/monolog@3.8.1", + "pkg:composer/monolog/monolog@3.8.0", + "pkg:composer/monolog/monolog@3.7.0", + "pkg:composer/monolog/monolog@3.6.0", + "pkg:composer/monolog/monolog@3.5.0", + "pkg:composer/monolog/monolog@3.4.0", + "pkg:composer/monolog/monolog@3.3.1", + "pkg:composer/monolog/monolog@3.3.0", + "pkg:composer/monolog/monolog@3.2.0", + "pkg:composer/monolog/monolog@3.1.0", + "pkg:composer/monolog/monolog@3.0.0", + "pkg:composer/monolog/monolog@3.0.0-RC1", + "pkg:composer/monolog/monolog@2.10.0", + "pkg:composer/monolog/monolog@2.9.3", + "pkg:composer/monolog/monolog@2.9.2", + "pkg:composer/monolog/monolog@2.9.1", + "pkg:composer/monolog/monolog@2.9.0", + "pkg:composer/monolog/monolog@2.8.0", + "pkg:composer/monolog/monolog@2.7.0", + "pkg:composer/monolog/monolog@2.6.0", + "pkg:composer/monolog/monolog@2.5.0", + "pkg:composer/monolog/monolog@2.4.0", + "pkg:composer/monolog/monolog@2.3.5", + "pkg:composer/monolog/monolog@2.3.4", + "pkg:composer/monolog/monolog@2.3.3", + "pkg:composer/monolog/monolog@2.3.2", + "pkg:composer/monolog/monolog@2.3.1", + "pkg:composer/monolog/monolog@2.3.0", + "pkg:composer/monolog/monolog@2.2.0", + "pkg:composer/monolog/monolog@2.1.1", + "pkg:composer/monolog/monolog@2.1.0", + "pkg:composer/monolog/monolog@2.0.2", + "pkg:composer/monolog/monolog@2.0.1", + "pkg:composer/monolog/monolog@2.0.0", + "pkg:composer/monolog/monolog@2.0.0-beta2", + "pkg:composer/monolog/monolog@2.0.0-beta1", + "pkg:composer/monolog/monolog@1.27.1", + "pkg:composer/monolog/monolog@1.27.0", + "pkg:composer/monolog/monolog@1.26.1", + "pkg:composer/monolog/monolog@1.26.0", + "pkg:composer/monolog/monolog@1.25.5", + "pkg:composer/monolog/monolog@1.25.4", + "pkg:composer/monolog/monolog@1.25.3", + "pkg:composer/monolog/monolog@1.25.2", + "pkg:composer/monolog/monolog@1.25.1", + "pkg:composer/monolog/monolog@1.25.0", + "pkg:composer/monolog/monolog@1.24.0", + "pkg:composer/monolog/monolog@1.23.0", + "pkg:composer/monolog/monolog@1.22.1", + "pkg:composer/monolog/monolog@1.22.0", + "pkg:composer/monolog/monolog@1.21.0", + "pkg:composer/monolog/monolog@1.20.0", + "pkg:composer/monolog/monolog@1.19.0", + "pkg:composer/monolog/monolog@1.18.2", + "pkg:composer/monolog/monolog@1.18.1", + "pkg:composer/monolog/monolog@1.18.0", + "pkg:composer/monolog/monolog@1.17.2", + "pkg:composer/monolog/monolog@1.17.1", + "pkg:composer/monolog/monolog@1.17.0", + "pkg:composer/monolog/monolog@1.16.0", + "pkg:composer/monolog/monolog@1.15.0", + "pkg:composer/monolog/monolog@1.14.0", + "pkg:composer/monolog/monolog@1.13.1", + "pkg:composer/monolog/monolog@1.13.0", + "pkg:composer/monolog/monolog@1.12.0", + "pkg:composer/monolog/monolog@1.11.0", + "pkg:composer/monolog/monolog@1.10.0", + "pkg:composer/monolog/monolog@1.9.1", + "pkg:composer/monolog/monolog@1.9.0", + "pkg:composer/monolog/monolog@1.8.0", + "pkg:composer/monolog/monolog@1.7.0", + "pkg:composer/monolog/monolog@1.6.0", + "pkg:composer/monolog/monolog@1.5.0", + "pkg:composer/monolog/monolog@1.4.1", + "pkg:composer/monolog/monolog@1.4.0", + "pkg:composer/monolog/monolog@1.3.1", + "pkg:composer/monolog/monolog@1.3.0", + "pkg:composer/monolog/monolog@1.2.1", + "pkg:composer/monolog/monolog@1.2.0", + "pkg:composer/monolog/monolog@1.1.0", + "pkg:composer/monolog/monolog@1.0.2", + "pkg:composer/monolog/monolog@1.0.1", + "pkg:composer/monolog/monolog@1.0.0", + "pkg:composer/monolog/monolog@1.0.0-RC1" +] \ No newline at end of file diff --git a/minecode_pipelines/tests/test_data/conan/cairo/cairo-config.yml b/minecode_pipelines/tests/test_data/conan/cairo-config.yml similarity index 100% rename from minecode_pipelines/tests/test_data/conan/cairo/cairo-config.yml rename to minecode_pipelines/tests/test_data/conan/cairo-config.yml diff --git a/pyproject-minecode_pipelines.toml b/pyproject-minecode_pipelines.toml index 85e20401..1d67d2f4 100644 --- a/pyproject-minecode_pipelines.toml +++ b/pyproject-minecode_pipelines.toml @@ -4,7 +4,7 @@ build-backend = "flot.buildapi" [project] name = "minecode_pipelines" -version = "0.0.1b59" +version = "0.0.1b26" description = "A library for mining packageURLs and package metadata from ecosystem repositories." readme = "minecode_pipelines/README.rst" license = { text = "Apache-2.0" } @@ -43,8 +43,7 @@ dependencies = [ "scancodeio >= 35.3.0", "ftputil >= 5.1.0", "jawa >= 2.2.0", - "arrow >= 1.3.0", - "beautifulsoup4 >= 4.13.4" + "arrow >= 1.3.0" ] urls = { Homepage = "https://github.com/aboutcode-org/purldb" } @@ -57,13 +56,12 @@ mine_debian = "minecode_pipelines.pipelines.mine_debian:MineDebian" mine_nuget = "minecode_pipelines.pipelines.mine_nuget:MineNuGet" mine_alpine = "minecode_pipelines.pipelines.mine_alpine:MineAlpine" mine_conan = "minecode_pipelines.pipelines.mine_conan:MineConan" -mine_cpan = "minecode_pipelines.pipelines.mine_cpan:MineCpan" mine_cran = "minecode_pipelines.pipelines.mine_cran:MineCran" mine_swift = "minecode_pipelines.pipelines.mine_swift:MineSwift" mine_composer = "minecode_pipelines.pipelines.mine_composer:MineComposer" [tool.bumpversion] -current_version = "0.0.1b59" +current_version = "0.0.1b25" allow_dirty = true files = [