Skip to content

Fix minecode_pipelines bugs #802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AyanSinhaMahapatra wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix minecode_pipelines bugs #802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions minecode_pipelines/pipelines/mine_pypi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ def mine_and_publish_packageurls(self):
def update_state_and_checkpoints(self):
pypi.update_state_and_checkpoints(
config_repo=self.config_repo,
state=self.state,
last_serial=self.last_serial,
logger=self.log,
)
5 changes: 0 additions & 5 deletions minecode_pipelines/pipes/npm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,6 @@
COMPRESSED_NPM_REPLICATE_CHECKPOINT_PATH = "npm/" + COMPRESSED_PACKAGE_FILE_NAME
NPM_CHECKPOINT_PATH = "npm/checkpoints.json"
NPM_PACKAGES_CHECKPOINT_PATH = "npm/packages_checkpoint.json"

# We are testing and storing mined packageURLs in one single repo per ecosystem for now
MINECODE_DATA_NPM_REPO = "https://github.com/aboutcode-data/minecode-data-npm-test"


PACKAGE_BATCH_SIZE = 700


Expand Down
7 changes: 1 addition & 6 deletions minecode_pipelines/pipes/pypi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
from minecode_pipelines.pipes import get_mined_packages_from_checkpoint
from minecode_pipelines.pipes import update_mined_packages_in_checkpoint
from minecode_pipelines.pipes import get_packages_file_from_checkpoint
from minecode_pipelines.pipes import update_checkpoint_state
from minecode_pipelines.pipes import write_packages_json
from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO
from minecode_pipelines.pipes import INITIAL_SYNC_STATE
Expand Down Expand Up @@ -58,10 +57,6 @@
PYPI_PACKAGES_CHECKPOINT_PATH = "pypi/packages_checkpoint.json"


# We are testing and storing mined packageURLs in one single repo per ecosystem for now
MINECODE_DATA_PYPI_REPO = "https://github.com/aboutcode-data/minecode-data-pypi-test"


def mine_pypi_packages(logger=None):
"""
Mine pypi package names from pypi simple and save to checkpoints,
Expand Down Expand Up @@ -277,7 +272,7 @@ def save_mined_packages_in_checkpoint(packages_mined, config_repo, logger=None):
)


def update_state_and_checkpoints(config_repo, last_serial, logger=None):
def update_state_and_checkpoints(config_repo, state, last_serial, logger=None):
# If we are finshed mining all the packages in the intial sync, we can now
# periodically sync the packages from latest
if state == INITIAL_SYNC_STATE:
Expand Down
Loading