From 5ab5d54cdb23b47d54781a88d7d8211c375c6847 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Fri, 12 Dec 2025 23:36:05 -0800 Subject: [PATCH 01/27] normalize `localId` padding and handle missing pricing data --- .../poke_cli_dbt/macros/create_view.sql | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/card_data/pipelines/poke_cli_dbt/macros/create_view.sql b/card_data/pipelines/poke_cli_dbt/macros/create_view.sql index c9b2952..1432e14 100644 --- a/card_data/pipelines/poke_cli_dbt/macros/create_view.sql +++ b/card_data/pipelines/poke_cli_dbt/macros/create_view.sql @@ -4,37 +4,37 @@ WITH cards_cte AS ( SELECT set_id, - image, name, + image, + illustrator, "localId", "set_cardCount_official", - CONCAT(name, ' - ', "localId", '/', LPAD("set_cardCount_official"::text, 3, '0')) AS card_combined_name, + CONCAT(name, ' - ', LPAD("localId", 3, '0'), '/', LPAD("set_cardCount_official"::text, 3, '0')) AS card_combined_name, set_name FROM public.cards ), - cards_pricing_cte AS ( SELECT product_id, market_price, - CONCAT(name, ' - ', card_number) AS card_combined_name, + CONCAT(REPLACE(name, ' (Secret)', ''), ' - ', card_number) AS card_combined_name, card_number FROM public.pricing_data ) - SELECT c.set_id, c.name, - CONCAT(p.card_number, ' - ', c.name) AS number_plus_name, + CONCAT(COALESCE(p.card_number, LPAD(c."localId", 3, '0')), ' - ', c.name) AS number_plus_name, CONCAT(c.image, '/high.png') AS image_url, c.set_name, - c."localId", + LPAD(c."localId", 3, '0') AS "localId", p."market_price", - p."card_number" + COALESCE(p."card_number", LPAD(c."localId", 3, '0')) AS card_number, + c.illustrator FROM cards_cte AS c - INNER JOIN + LEFT JOIN cards_pricing_cte AS p ON c.card_combined_name = p.card_combined_name - ORDER BY c."localId" + ORDER BY c."localId"::integer {% endmacro %} \ No newline at end of file From d754bd97528afb4efafc211f9b3e1e111913779c Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Fri, 12 Dec 2025 23:39:01 -0800 Subject: [PATCH 02/27] adding `swsh` and `sm` series (#212) --- card_data/pipelines/defs/extract/extract_data.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/card_data/pipelines/defs/extract/extract_data.py b/card_data/pipelines/defs/extract/extract_data.py index b523b1c..2dff930 100644 --- a/card_data/pipelines/defs/extract/extract_data.py +++ b/card_data/pipelines/defs/extract/extract_data.py @@ -42,16 +42,17 @@ def extract_series_data() -> pl.DataFrame: print(e) raise - filtered = [s.model_dump(mode="json") for s in validated if s.id in ["swsh", "sv", "me"]] + filtered = [s.model_dump(mode="json") for s in validated if s.id in ["sm", "swsh", "sv", "me"]] return pl.DataFrame(filtered) @dg.asset(kinds={"API", "Polars", "Pydantic"}, name="extract_set_data") def extract_set_data() -> pl.DataFrame: url_list = [ - "https://api.tcgdex.net/v2/en/series/swsh", - "https://api.tcgdex.net/v2/en/series/sv", "https://api.tcgdex.net/v2/en/series/me", + "https://api.tcgdex.net/v2/en/series/sv", + "https://api.tcgdex.net/v2/en/series/swsh", + "https://api.tcgdex.net/v2/en/series/sm", ] flat: list[dict] = [] From 9dd4b7266546fdb007bf8a697f6fc91c5be1d895 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Fri, 12 Dec 2025 23:43:05 -0800 Subject: [PATCH 03/27] updating soda data quality rule --- card_data/pipelines/soda/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/card_data/pipelines/soda/checks.yml b/card_data/pipelines/soda/checks.yml index 95d4b1a..ef3cf7d 100644 --- a/card_data/pipelines/soda/checks.yml +++ b/card_data/pipelines/soda/checks.yml @@ -1,6 +1,6 @@ checks for series: # Row count validation - - row_count = 3 + - row_count = 4 # Schema validation checks - schema: From e2227b91755a349acbf8a3824dd7557f74be3c02 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sat, 13 Dec 2025 23:52:34 -0800 Subject: [PATCH 04/27] fixing duplicate API call issue --- card_data/pipelines/defs/load/load_data.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/card_data/pipelines/defs/load/load_data.py b/card_data/pipelines/defs/load/load_data.py index 8ec9a9e..092dff1 100644 --- a/card_data/pipelines/defs/load/load_data.py +++ b/card_data/pipelines/defs/load/load_data.py @@ -10,19 +10,19 @@ from termcolor import colored import subprocess from pathlib import Path +import polars as pl @dg.asset( - deps=[extract_series_data], kinds={"Supabase", "Postgres"}, name="load_series_data", retry_policy=RetryPolicy(max_retries=3, delay=2, backoff=Backoff.EXPONENTIAL) ) -def load_series_data() -> None: +def load_series_data(extract_series_data: pl.DataFrame) -> None: database_url: str = fetch_secret() table_name: str = "staging.series" - df = extract_series_data() + df = extract_series_data try: df.write_database( table_name=table_name, connection=database_url, if_table_exists="replace" @@ -68,16 +68,15 @@ def data_quality_check_on_series() -> None: @dg.asset( - deps=[extract_set_data], kinds={"Supabase", "Postgres"}, name="load_set_data", retry_policy=RetryPolicy(max_retries=3, delay=2, backoff=Backoff.EXPONENTIAL) ) -def load_set_data() -> None: +def load_set_data(extract_set_data: pl.DataFrame) -> None: database_url: str = fetch_secret() table_name: str = "staging.sets" - df = extract_set_data() + df = extract_set_data try: df.write_database( table_name=table_name, connection=database_url, if_table_exists="replace" @@ -89,16 +88,15 @@ def load_set_data() -> None: @dg.asset( - deps=[create_card_dataframe], kinds={"Supabase", "Postgres"}, name="load_card_data", retry_policy=RetryPolicy(max_retries=3, delay=2, backoff=Backoff.EXPONENTIAL) ) -def load_card_data() -> None: +def load_card_data(create_card_dataframe: pl.DataFrame) -> None: database_url: str = fetch_secret() table_name: str = "staging.cards" - df = create_card_dataframe() + df = create_card_dataframe try: df.write_database( table_name=table_name, connection=database_url, if_table_exists="append" From 2fb3452b2d897ebece8d00a4a212448f3ba72ad5 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 11:39:36 -0800 Subject: [PATCH 05/27] adding Sword & Shield to series list (#212) --- cmd/card/serieslist.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/card/serieslist.go b/cmd/card/serieslist.go index 520fc6e..ef6e1c3 100644 --- a/cmd/card/serieslist.go +++ b/cmd/card/serieslist.go @@ -63,6 +63,7 @@ func SeriesList() SeriesModel { items := []list.Item{ item("Mega Evolution"), item("Scarlet & Violet"), + item("Sword & Shield"), } const listWidth = 20 From 81f786a5452313803f98281bb7487089924fdeca Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 12:56:52 -0800 Subject: [PATCH 06/27] adding paths to ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 30e3fbd..c4fae95 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,4 @@ card_data/pipelines/poke_cli_dbt/.user.yml /card_data/sample_scripts/ card_data/~/ +card_data/storage/ From 0d1c8e46a767bfd032b630d12f2d75e28f8abf4d Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 12:59:01 -0800 Subject: [PATCH 07/27] updating version numbers --- .github/workflows/ci.yml | 2 +- .goreleaser.yml | 2 +- Dockerfile | 2 +- README.md | 6 +++--- nfpm.yaml | 2 +- testdata/main_latest_flag.golden | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8988a2b..13e130a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ on: - main env: - VERSION_NUMBER: 'v1.8.1' + VERSION_NUMBER: 'v1.8.2' DOCKERHUB_REGISTRY_NAME: 'digitalghostdev/poke-cli' AWS_REGION: 'us-west-2' diff --git a/.goreleaser.yml b/.goreleaser.yml index 88a2427..8a53370 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -14,7 +14,7 @@ builds: - windows - darwin ldflags: - - -s -w -X main.version=v1.8.1 + - -s -w -X main.version=v1.8.2 archives: - formats: [ 'zip' ] diff --git a/Dockerfile b/Dockerfile index 327d079..fd0cfb0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN go mod download COPY . . -RUN go build -ldflags "-X main.version=v1.8.1" -o poke-cli . +RUN go build -ldflags "-X main.version=v1.8.2" -o poke-cli . # build 2 FROM --platform=$BUILDPLATFORM alpine:3.22 diff --git a/README.md b/README.md index 5eef998..f29c551 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ pokemon-logo

Pokémon CLI

version-label - docker-image-size + docker-image-size ci-status-badge
@@ -95,11 +95,11 @@ Cloudsmith is a fully cloud-based service that lets you easily create, store, an 3. Choose how to interact with the container: * Run a single command and exit: ```bash - docker run --rm -it digitalghostdev/poke-cli:v1.8.1 [subcommand] [flag] + docker run --rm -it digitalghostdev/poke-cli:v1.8.2 [subcommand] [flag] ``` * Enter the container and use its shell: ```bash - docker run --rm -it --name poke-cli --entrypoint /bin/sh digitalghostdev/poke-cli:v1.8.1 -c "cd /app && exec sh" + docker run --rm -it --name poke-cli --entrypoint /bin/sh digitalghostdev/poke-cli:v1.8.2 -c "cd /app && exec sh" # placed into the /app directory, run the program with './poke-cli' # example: ./poke-cli ability swift-swim ``` diff --git a/nfpm.yaml b/nfpm.yaml index dfea760..84aabaa 100644 --- a/nfpm.yaml +++ b/nfpm.yaml @@ -1,7 +1,7 @@ name: "poke-cli" arch: "arm64" platform: "linux" -version: "v1.8.1" +version: "v1.8.2" section: "default" version_schema: semver maintainer: "Christian S" diff --git a/testdata/main_latest_flag.golden b/testdata/main_latest_flag.golden index ebd0a2a..bfac272 100644 --- a/testdata/main_latest_flag.golden +++ b/testdata/main_latest_flag.golden @@ -2,6 +2,6 @@ ┃ ┃ ┃ Latest available release ┃ ┃ on GitHub: ┃ -┃ • v1.8.0 ┃ +┃ • v1.8.1 ┃ ┃ ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ From ce1cc4eef5fdf29d16228ca54500065e61f63840 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 12:59:39 -0800 Subject: [PATCH 08/27] adding regex_replace --- card_data/pipelines/poke_cli_dbt/macros/create_view.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/card_data/pipelines/poke_cli_dbt/macros/create_view.sql b/card_data/pipelines/poke_cli_dbt/macros/create_view.sql index 1432e14..5e940f1 100644 --- a/card_data/pipelines/poke_cli_dbt/macros/create_view.sql +++ b/card_data/pipelines/poke_cli_dbt/macros/create_view.sql @@ -17,7 +17,7 @@ SELECT product_id, market_price, - CONCAT(REPLACE(name, ' (Secret)', ''), ' - ', card_number) AS card_combined_name, + CONCAT(REGEXP_REPLACE(name, '\s*\(.*\)$', ''), ' - ', card_number) AS card_combined_name, card_number FROM public.pricing_data ) From acf0ff2e70bdb02dfbb8a5b13654a35e0ef85e6b Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 13:01:30 -0800 Subject: [PATCH 09/27] adding better messaging to missing data (#213) --- cmd/card/cardlist.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/card/cardlist.go b/cmd/card/cardlist.go index ac86f22..cb48743 100644 --- a/cmd/card/cardlist.go +++ b/cmd/card/cardlist.go @@ -119,8 +119,18 @@ func CardsList(setID string) (CardsModel, error) { illustratorMap := make(map[string]string) for i, card := range allCards { rows[i] = []string{card.NumberPlusName} - priceMap[card.NumberPlusName] = fmt.Sprintf("Price: $%.2f", card.MarketPrice) - illustratorMap[card.NumberPlusName] = "Illustrator: " + card.Illustrator + if card.MarketPrice != 0 { + priceMap[card.NumberPlusName] = fmt.Sprintf("Price: $%.2f", card.MarketPrice) + } else { + priceMap[card.NumberPlusName] = "Pricing not available" + } + + if card.Illustrator != "" { + illustratorMap[card.NumberPlusName] = "Illustrator: " + card.Illustrator + } else { + illustratorMap[card.NumberPlusName] = "Illustrator not available" + } + imageMap[card.NumberPlusName] = card.ImageURL } From 912e66581ae6b93208a8ecb614a7f1136fc09f76 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 15:59:44 -0800 Subject: [PATCH 10/27] updating version numbers --- card_data/pipelines/poke_cli_dbt/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/card_data/pipelines/poke_cli_dbt/dbt_project.yml b/card_data/pipelines/poke_cli_dbt/dbt_project.yml index 333cfe3..2f56743 100644 --- a/card_data/pipelines/poke_cli_dbt/dbt_project.yml +++ b/card_data/pipelines/poke_cli_dbt/dbt_project.yml @@ -1,5 +1,5 @@ name: 'poke_cli_dbt' -version: '1.8.1' +version: '1.8.2' profile: 'poke_cli_dbt' From 41177f9578a01b8a641de08b64136efb88d2aa79 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 16:00:52 -0800 Subject: [PATCH 11/27] updating to Go `v1.24.11` (#214) --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 920a8a2..a4c89cc 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/digitalghost-dev/poke-cli -go 1.24.9 +go 1.24.11 require ( github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 From 09a770c1b272039c1bf1c85bd6969547da7f827b Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 16:04:39 -0800 Subject: [PATCH 12/27] updating base `golang` image (#215) --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index fd0cfb0..f97a844 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # build 1 -FROM golang:1.24.9-alpine3.22 AS build +FROM golang:1.24.11-alpine3.23 AS build WORKDIR /app @@ -11,10 +11,10 @@ COPY . . RUN go build -ldflags "-X main.version=v1.8.2" -o poke-cli . # build 2 -FROM --platform=$BUILDPLATFORM alpine:3.22 +FROM --platform=$BUILDPLATFORM alpine:3.23 # Installing only necessary packages and remove them after use -RUN apk add --no-cache shadow=4.17.3-r0 && \ +RUN apk add --no-cache shadow=4.18.0-r0 && \ addgroup -S poke_group && adduser -S poke_user -G poke_group && \ sed -i 's/^root:.*/root:!*:0:0:root:\/root:\/sbin\/nologin/' /etc/passwd && \ apk del shadow From 52d0586530a005d6de0d7b21086ca41f4fe681aa Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 19:20:08 -0800 Subject: [PATCH 13/27] fixing duplicate API call issue --- card_data/pipelines/defs/extract/extract_data.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/card_data/pipelines/defs/extract/extract_data.py b/card_data/pipelines/defs/extract/extract_data.py index 2dff930..4b6d72a 100644 --- a/card_data/pipelines/defs/extract/extract_data.py +++ b/card_data/pipelines/defs/extract/extract_data.py @@ -114,9 +114,9 @@ def extract_card_url_from_set() -> list: return all_card_urls -@dg.asset(deps=[extract_card_url_from_set], kinds={"API"}, name="extract_card_info") -def extract_card_info() -> list: - card_url_list = extract_card_url_from_set() +@dg.asset(kinds={"API"}, name="extract_card_info") +def extract_card_info(extract_card_url_from_set_data: list) -> list: + card_url_list = extract_card_url_from_set_data cards_list = [] for url in card_url_list: @@ -125,7 +125,7 @@ def extract_card_info() -> list: r.raise_for_status() data = r.json() cards_list.append(data) - # print(f"Retrieved card: {data['id']} - {data.get('name', 'Unknown')}") + print(f"Retrieved card: {data['id']} - {data.get('name', 'Unknown')}") time.sleep(0.1) except requests.RequestException as e: print(f"Failed to fetch {url}: {e}") @@ -133,9 +133,9 @@ def extract_card_info() -> list: return cards_list -@dg.asset(deps=[extract_card_info], kinds={"Polars"}, name="create_card_dataframe") -def create_card_dataframe() -> pl.DataFrame: - cards_list = extract_card_info() +@dg.asset(kinds={"Polars"}, name="create_card_dataframe") +def create_card_dataframe(extract_card_info: list) -> pl.DataFrame: + cards_list = extract_card_info all_flat_cards = [] From c884cbace52df67a380a12a9bd0746f4c156cc4e Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 19:22:26 -0800 Subject: [PATCH 14/27] focusing on sword & shield for `v1.8.2` (#212) --- card_data/pipelines/defs/extract/extract_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/card_data/pipelines/defs/extract/extract_data.py b/card_data/pipelines/defs/extract/extract_data.py index 4b6d72a..217ba49 100644 --- a/card_data/pipelines/defs/extract/extract_data.py +++ b/card_data/pipelines/defs/extract/extract_data.py @@ -42,7 +42,7 @@ def extract_series_data() -> pl.DataFrame: print(e) raise - filtered = [s.model_dump(mode="json") for s in validated if s.id in ["sm", "swsh", "sv", "me"]] + filtered = [s.model_dump(mode="json") for s in validated if s.id in ["swsh", "sv", "me"]] return pl.DataFrame(filtered) @@ -52,7 +52,6 @@ def extract_set_data() -> pl.DataFrame: "https://api.tcgdex.net/v2/en/series/me", "https://api.tcgdex.net/v2/en/series/sv", "https://api.tcgdex.net/v2/en/series/swsh", - "https://api.tcgdex.net/v2/en/series/sm", ] flat: list[dict] = [] From acacae42664a9491feb7dc3935bb9abe7b9efbce Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 19:23:48 -0800 Subject: [PATCH 15/27] filtering out Trainer Gallery cards for the time being --- card_data/pipelines/defs/extract/extract_data.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/card_data/pipelines/defs/extract/extract_data.py b/card_data/pipelines/defs/extract/extract_data.py index 217ba49..45c1406 100644 --- a/card_data/pipelines/defs/extract/extract_data.py +++ b/card_data/pipelines/defs/extract/extract_data.py @@ -102,7 +102,11 @@ def extract_card_url_from_set() -> list: data = r.json()["cards"] - set_card_urls = [f"https://api.tcgdex.net/v2/en/cards/{card['id']}" for card in data] + set_card_urls = [ + f"https://api.tcgdex.net/v2/en/cards/{card['id']}" + for card in data + if '-TG' not in card['id'] + ] all_card_urls.extend(set_card_urls) time.sleep(0.1) From c5963b89b63268bf1a7c1a380f54aed489995603 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 19:28:47 -0800 Subject: [PATCH 16/27] filtering out text/numbers in parentheses within a card name --- .../defs/extract/extract_pricing_data.py | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/card_data/pipelines/defs/extract/extract_pricing_data.py b/card_data/pipelines/defs/extract/extract_pricing_data.py index 377af30..4aef703 100644 --- a/card_data/pipelines/defs/extract/extract_pricing_data.py +++ b/card_data/pipelines/defs/extract/extract_pricing_data.py @@ -1,4 +1,6 @@ from typing import Optional +import re +import unicodedata import dagster as dg import polars as pl @@ -53,8 +55,34 @@ def get_card_number(card: dict) -> Optional[str]: def extract_card_name(full_name: str) -> str: - """Extract clean card name, removing variant information after dash""" - return full_name.partition("-")[0].strip() if "-" in full_name else full_name + """Extract clean card name, removing variant information after dash and parenthetical suffixes""" + + name = full_name.partition("-")[0].strip() if "-" in full_name else full_name + + # Remove parenthetical card numbers like "(010)" or "(1)" + # Pattern: space followed by parentheses containing only digits + name = re.sub(r"\s+\(\d+\)$", "", name) + + # Remove known variant types in parentheses + # e.g., "(Secret)", "(Full Art)", "(Reverse Holofoil)", etc. + variant_types = [ + "Poke Ball Pattern", + "Master Ball Pattern", + "Full Art", + "Secret", + "Reverse Holofoil", + "Rainbow Rare", + "Gold", + ] + for variant in variant_types: + name = name.replace(f" ({variant})", "") + + # Normalize accented characters (é → e, ñ → n, etc.) + # NFD decomposes characters into base + diacritics, then we filter out diacritics + name = unicodedata.normalize("NFD", name) + name = "".join(char for char in name if unicodedata.category(char) != "Mn") + + return name.strip() def pull_product_information(set_number: str) -> pl.DataFrame: From 60711d189648ee47d381e2007094dfee810f8a01 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 19:32:12 -0800 Subject: [PATCH 17/27] adding sword & shield product mapping (#212) --- .../defs/extract/extract_pricing_data.py | 50 +++++++++++++------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/card_data/pipelines/defs/extract/extract_pricing_data.py b/card_data/pipelines/defs/extract/extract_pricing_data.py index 4aef703..cf5696b 100644 --- a/card_data/pipelines/defs/extract/extract_pricing_data.py +++ b/card_data/pipelines/defs/extract/extract_pricing_data.py @@ -10,24 +10,42 @@ SET_PRODUCT_MATCHING = { - "sv01": "22873", - "sv02": "23120", - "sv03": "23228", - "sv03.5": "23237", - "sv04": "23286", - "sv04.5": "23353", - "sv05": "23381", - "sv06": "23473", - "sv06.5": "23529", - "sv07": "23537", - "sv08": "23651", - "sv08.5": "23821", - "sv09": "24073", - "sv10": "24269", + "me02": "24448", + "me01": "24380", + # Scarlet & Violet "sv10.5b": "24325", "sv10.5w": "24326", - "me01": "24380", - "me02": "24448" + "sv10": "24269", + "sv09": "24073", + "sv08.5": "23821", + "sv08": "23651", + "sv07": "23537", + "sv06.5": "23529", + "sv06": "23473", + "sv05": "23381", + "sv04.5": "23353", + "sv04": "23286", + "sv03.5": "23237", + "sv03": "23228", + "sv02": "23120", + "sv01": "22873", + # Sword & Shield + "swsh12.5": "17688", + "swsh12": "3170", + "swsh11": "3118", + "swsh10.5": "3064", + "swsh10": "3040", + "swsh9": "2948", + "swsh8": "2906", + "swsh7": "2848", + "swsh6": "2807", + "swsh5": "2765", + "swsh4.5": "2754", + "swsh4": "2701", + "swsh3.5": "2685", + "swsh3": "2675", + "swsh2": "2626", + "swsh1": "2585", } From 31c9d9eac9922f79d8e943892ffdcf9cd5e6cf75 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 19:37:38 -0800 Subject: [PATCH 18/27] applying `ruff` formatting --- .../pipelines/defs/extract/extract_data.py | 35 ++++++++++++------- .../defs/extract/extract_pricing_data.py | 6 ++-- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/card_data/pipelines/defs/extract/extract_data.py b/card_data/pipelines/defs/extract/extract_data.py index 45c1406..2e95051 100644 --- a/card_data/pipelines/defs/extract/extract_data.py +++ b/card_data/pipelines/defs/extract/extract_data.py @@ -10,6 +10,7 @@ import requests + class Series(BaseModel): id: str name: str @@ -42,7 +43,9 @@ def extract_series_data() -> pl.DataFrame: print(e) raise - filtered = [s.model_dump(mode="json") for s in validated if s.id in ["swsh", "sv", "me"]] + filtered = [ + s.model_dump(mode="json") for s in validated if s.id in ["swsh", "sv", "me"] + ] return pl.DataFrame(filtered) @@ -68,17 +71,14 @@ def extract_set_data() -> pl.DataFrame: "official_card_count": s.get("cardCount", {}).get("official"), "total_card_count": s.get("cardCount", {}).get("total"), "logo": s.get("logo"), - "symbol": s.get("symbol") + "symbol": s.get("symbol"), } flat.append(entry) # Pydantic validation try: validated: list[Set] = [Set(**item) for item in flat] - print( - colored(" ✓", "green"), - "Pydantic validation passed for all set entries." - ) + print(colored(" ✓", "green"), "Pydantic validation passed for all set entries.") except ValidationError as e: print(colored(" ✖", "red"), "Pydantic validation failed.") print(e) @@ -89,9 +89,7 @@ def extract_set_data() -> pl.DataFrame: @dg.asset(kinds={"API"}, name="extract_card_url_from_set_data") def extract_card_url_from_set() -> list: - urls = [ - "https://api.tcgdex.net/v2/en/sets/me02" - ] + urls = ["https://api.tcgdex.net/v2/en/sets/me02"] all_card_urls = [] @@ -105,7 +103,7 @@ def extract_card_url_from_set() -> list: set_card_urls = [ f"https://api.tcgdex.net/v2/en/cards/{card['id']}" for card in data - if '-TG' not in card['id'] + if "-TG" not in card["id"] ] all_card_urls.extend(set_card_urls) @@ -146,8 +144,19 @@ def create_card_dataframe(extract_card_info: list) -> pl.DataFrame: flat = {} # Copy top-level scalar values - scalar_keys = ['category', 'hp', 'id', 'illustrator', 'image', 'localId', - 'name', 'rarity', 'regulationMark', 'retreat', 'stage'] + scalar_keys = [ + "category", + "hp", + "id", + "illustrator", + "image", + "localId", + "name", + "rarity", + "regulationMark", + "retreat", + "stage", + ] for key in scalar_keys: flat[key] = card.get(key) @@ -169,7 +178,7 @@ def create_card_dataframe(extract_card_info: list) -> pl.DataFrame: attacks = card.get("attacks", []) for i, atk in enumerate(attacks): - prefix = f"attack_{i+1}" + prefix = f"attack_{i + 1}" flat[f"{prefix}_name"] = atk.get("name") flat[f"{prefix}_damage"] = atk.get("damage") flat[f"{prefix}_effect"] = atk.get("effect") diff --git a/card_data/pipelines/defs/extract/extract_pricing_data.py b/card_data/pipelines/defs/extract/extract_pricing_data.py index cf5696b..bf8699f 100644 --- a/card_data/pipelines/defs/extract/extract_pricing_data.py +++ b/card_data/pipelines/defs/extract/extract_pricing_data.py @@ -161,8 +161,10 @@ def build_dataframe() -> pl.DataFrame: # Raise error if any DataFrame is empty if df is None or df.shape[1] == 0 or df.is_empty(): - error_msg = f"Empty DataFrame returned for set '{set_number}'. " \ - f"Cannot proceed with drop+replace operation to avoid data loss." + error_msg = ( + f"Empty DataFrame returned for set '{set_number}'. " + f"Cannot proceed with drop+replace operation to avoid data loss." + ) print(colored(" ✖", "red"), error_msg) raise ValueError(error_msg) From b6a9b5a9578cf30a3100ccf7cba60dd21ab2fdfb Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 20:10:59 -0800 Subject: [PATCH 19/27] adding dependencies --- card_data/pyproject.toml | 4 +++- card_data/uv.lock | 52 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/card_data/pyproject.toml b/card_data/pyproject.toml index 5f2118a..62cdcfb 100644 --- a/card_data/pyproject.toml +++ b/card_data/pyproject.toml @@ -19,7 +19,9 @@ dependencies = [ "psycopg2-binary>=2.9.10", "pyarrow>=20.0.0", "pydantic>=2.11.7", + "pytest>=9.0.2", "requests>=2.32.4", + "responses>=0.25.8", "soda-core-postgres>=3.5.5", "sqlalchemy>=2.0.41", "termcolor>=3.1.0", @@ -46,4 +48,4 @@ registry_modules = [ override-dependencies = [ "deepdiff==8.6.1", "starlette==0.49.1", -] \ No newline at end of file +] diff --git a/card_data/uv.lock b/card_data/uv.lock index aeae284..ce60f02 100644 --- a/card_data/uv.lock +++ b/card_data/uv.lock @@ -157,7 +157,9 @@ dependencies = [ { name = "psycopg2-binary" }, { name = "pyarrow" }, { name = "pydantic" }, + { name = "pytest" }, { name = "requests" }, + { name = "responses" }, { name = "soda-core-postgres" }, { name = "sqlalchemy" }, { name = "termcolor" }, @@ -187,7 +189,9 @@ requires-dist = [ { name = "psycopg2-binary", specifier = ">=2.9.10" }, { name = "pyarrow", specifier = ">=20.0.0" }, { name = "pydantic", specifier = ">=2.11.7" }, + { name = "pytest", specifier = ">=9.0.2" }, { name = "requests", specifier = ">=2.32.4" }, + { name = "responses", specifier = ">=0.25.8" }, { name = "soda-core-postgres", specifier = ">=3.5.5" }, { name = "sqlalchemy", specifier = ">=2.0.41" }, { name = "termcolor", specifier = ">=3.1.0" }, @@ -981,6 +985,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/eb/427ed2b20a38a4ee29f24dbe4ae2dafab198674fe9a85e3d6adf9e5f5f41/inflect-7.5.0-py3-none-any.whl", hash = "sha256:2aea70e5e70c35d8350b8097396ec155ffd68def678c7ff97f51aa69c1d92344", size = 35197, upload-time = "2024-12-28T17:11:15.931Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "isodate" version = "0.6.1" @@ -1498,6 +1511,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, ] +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "polars" version = "1.31.0" @@ -1769,6 +1791,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/dc/491b7661614ab97483abf2056be1deee4dc2490ecbf7bff9ab5cdbac86e1/pyreadline3-3.5.4-py3-none-any.whl", hash = "sha256:eaf8e6cc3c49bcccf145fc6067ba8643d1df34d604a1ec0eccbf7a18e6d3fae6", size = 83178, upload-time = "2024-09-19T02:40:08.598Z" }, ] +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1915,6 +1953,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, ] +[[package]] +name = "responses" +version = "0.25.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/95/89c054ad70bfef6da605338b009b2e283485835351a9935c7bfbfaca7ffc/responses-0.25.8.tar.gz", hash = "sha256:9374d047a575c8f781b94454db5cab590b6029505f488d12899ddb10a4af1cf4", size = 79320, upload-time = "2025-08-08T19:01:46.709Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/4c/cc276ce57e572c102d9542d383b2cfd551276581dc60004cb94fe8774c11/responses-0.25.8-py3-none-any.whl", hash = "sha256:0c710af92def29c8352ceadff0c3fe340ace27cf5af1bbe46fb71275bcd2831c", size = 34769, upload-time = "2025-08-08T19:01:45.018Z" }, +] + [[package]] name = "rich" version = "14.1.0" From ce31b9a5c2522f1900c7c12cc58b18f72c9f2aef Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 20:11:29 -0800 Subject: [PATCH 20/27] adding test (#216) --- card_data/pipelines/defs/extract/testing.py | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 card_data/pipelines/defs/extract/testing.py diff --git a/card_data/pipelines/defs/extract/testing.py b/card_data/pipelines/defs/extract/testing.py new file mode 100644 index 0000000..fcdbf02 --- /dev/null +++ b/card_data/pipelines/defs/extract/testing.py @@ -0,0 +1,35 @@ +import pytest +import polars as pl +import responses +from extract_data import extract_series_data + +@pytest.fixture +def mock_api_response(): + """Sample API response matching tcgdex format""" + return [ + {"id": "sv", "name": "Scarlet & Violet", "logo": "https://example.com/sv.png"}, + {"id": "swsh", "name": "Sword & Shield", "logo": "https://example.com/swsh.png"}, + {"id": "xy", "name": "XY", "logo": "https://example.com/xy.png"}, # Should be filtered out + {"id": "me", "name": "McDonald's Collection", "logo": "https://example.com/me.png"}, + {"id": "sm", "name": "Sun & Moon", "logo": None}, # Should be filtered out + ] + +@responses.activate +def test_extract_series_data_success(mock_api_response): + """Test successful extraction and filtering""" + # Mock the API call + responses.add( + responses.GET, + "https://api.tcgdex.net/v2/en/series", + json=mock_api_response, + status=200 + ) + + result = extract_series_data() + + # Assertions + assert isinstance(result, pl.DataFrame) + assert len(result) == 3 # Only swsh, sv, me + assert set(result["id"].to_list()) == {"swsh", "sv", "me"} + assert "name" in result.columns + assert "logo" in result.columns \ No newline at end of file From a5f59cb648de39e506d17501b543d294948f0d66 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 20:23:05 -0800 Subject: [PATCH 21/27] moving testing libraries to dev dependency group --- card_data/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/card_data/pyproject.toml b/card_data/pyproject.toml index 62cdcfb..bb4b679 100644 --- a/card_data/pyproject.toml +++ b/card_data/pyproject.toml @@ -19,9 +19,7 @@ dependencies = [ "psycopg2-binary>=2.9.10", "pyarrow>=20.0.0", "pydantic>=2.11.7", - "pytest>=9.0.2", "requests>=2.32.4", - "responses>=0.25.8", "soda-core-postgres>=3.5.5", "sqlalchemy>=2.0.41", "termcolor>=3.1.0", @@ -33,6 +31,8 @@ dev = [ "dagster-dg-cli", "dagster-dbt>=0.27.3", "dagster-postgres>=0.27.3", + "pytest>=9.0.2", + "responses>=0.25.8", ] [tool.dg] From 85b5d9c58821e1ff07d7d88f38adb534be3ece8b Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 14 Dec 2025 20:48:07 -0800 Subject: [PATCH 22/27] updating roadmap checklist --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f29c551..43444b9 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ Below is a list of the planned/completed commands and flags: - [ ] `card`: get data about a TCG card. - [x] add mega evolution data - [x] add scarlet & violet data - - [ ] add sword & shield data + - [x] add sword & shield data - [ ] add sun & moon data - [ ] add x & y data - [x] `item`: get data about an item. From 63bd440f28009d3501a95d78f7cca5556f787270 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Mon, 15 Dec 2025 09:51:57 -0800 Subject: [PATCH 23/27] updating tests --- card_data/pipelines/defs/extract/testing.py | 2 +- cmd/card/cardlist.go | 5 +- cmd/card/cardlist_test.go | 111 ++++++++++++++++++++ cmd/card/serieslist_test.go | 23 ++++ 4 files changed, 139 insertions(+), 2 deletions(-) diff --git a/card_data/pipelines/defs/extract/testing.py b/card_data/pipelines/defs/extract/testing.py index fcdbf02..92a47f4 100644 --- a/card_data/pipelines/defs/extract/testing.py +++ b/card_data/pipelines/defs/extract/testing.py @@ -1,7 +1,7 @@ import pytest import polars as pl import responses -from extract_data import extract_series_data +from .extract_data import extract_series_data @pytest.fixture def mock_api_response(): diff --git a/cmd/card/cardlist.go b/cmd/card/cardlist.go index cb48743..7fc38ff 100644 --- a/cmd/card/cardlist.go +++ b/cmd/card/cardlist.go @@ -101,7 +101,7 @@ type cardData struct { // CardsList creates and returns a new CardsModel with cards from a specific set func CardsList(setID string) (CardsModel, error) { url := fmt.Sprintf("https://uoddayfnfkebrijlpfbh.supabase.co/rest/v1/card_pricing_view?set_id=eq.%s&select=number_plus_name,market_price,image_url,illustrator&order=localId", setID) - body, err := CallCardData(url) + body, err := getCardData(url) if err != nil { return CardsModel{}, fmt.Errorf("failed to fetch card data: %w", err) } @@ -159,6 +159,9 @@ func CardsList(setID string) (CardsModel, error) { }, nil } +// creating a function variable to swap the implementation in tests +var getCardData = CallCardData + func CallCardData(url string) ([]byte, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { diff --git a/cmd/card/cardlist_test.go b/cmd/card/cardlist_test.go index 7df0594..322bd0b 100644 --- a/cmd/card/cardlist_test.go +++ b/cmd/card/cardlist_test.go @@ -1,6 +1,7 @@ package card import ( + "errors" "strings" "testing" @@ -254,3 +255,113 @@ func TestCardsModel_View_MissingPrice(t *testing.T) { t.Error("View() should display 'Price: Not available' for cards without pricing") } } + +func TestCardsList_SuccessAndFallbacks(t *testing.T) { + // Save and restore getCardData stub + original := getCardData + defer func() { getCardData = original }() + + var capturedURL string + getCardData = func(url string) ([]byte, error) { + capturedURL = url + // Return two cards: one with price + illustrator, one with fallbacks + json := `[ + {"number_plus_name":"001/198 - Bulbasaur","market_price":1.5,"image_url":"https://example.com/bulba.png","illustrator":"Ken Sugimori"}, + {"number_plus_name":"002/198 - Ivysaur","market_price":0,"image_url":"https://example.com/ivy.png","illustrator":""} + ]` + return []byte(json), nil + } + + model, err := CardsList("set123") + if err != nil { + t.Fatalf("CardsList returned error: %v", err) + } + + // URL should target the correct set id and select fields + if !strings.Contains(capturedURL, "set_id=eq.set123") { + t.Errorf("expected URL to contain set_id filter, got %s", capturedURL) + } + if !strings.Contains(capturedURL, "select=number_plus_name,market_price,image_url,illustrator") { + t.Errorf("expected URL to contain select fields, got %s", capturedURL) + } + + // PriceMap expectations + if got := model.PriceMap["001/198 - Bulbasaur"]; got != "Price: $1.50" { + t.Errorf("unexpected price for Bulbasaur: %s", got) + } + if got := model.PriceMap["002/198 - Ivysaur"]; got != "Pricing not available" { + t.Errorf("unexpected price for Ivysaur: %s", got) + } + + // IllustratorMap expectations + if got := model.IllustratorMap["001/198 - Bulbasaur"]; got != "Illustrator: Ken Sugimori" { + t.Errorf("unexpected illustrator for Bulbasaur: %s", got) + } + if got := model.IllustratorMap["002/198 - Ivysaur"]; got != "Illustrator not available" { + t.Errorf("unexpected illustrator for Ivysaur: %s", got) + } + + // Image map + if model.ImageMap["001/198 - Bulbasaur"] != "https://example.com/bulba.png" { + t.Errorf("unexpected image url for Bulbasaur: %s", model.ImageMap["001/198 - Bulbasaur"]) + } + if model.ImageMap["002/198 - Ivysaur"] != "https://example.com/ivy.png" { + t.Errorf("unexpected image url for Ivysaur: %s", model.ImageMap["002/198 - Ivysaur"]) + } + + if row := model.Table.SelectedRow(); len(row) == 0 { + if model.View() == "" { + t.Error("model view should render even if no row is selected") + } + } +} + +func TestCardsList_FetchError(t *testing.T) { + original := getCardData + defer func() { getCardData = original }() + + getCardData = func(url string) ([]byte, error) { + return nil, errors.New("network error") + } + + _, err := CardsList("set123") + if err == nil { + t.Fatal("expected error when fetch fails") + } +} + +func TestCardsList_BadJSON(t *testing.T) { + original := getCardData + defer func() { getCardData = original }() + + getCardData = func(url string) ([]byte, error) { + return []byte("not-json"), nil + } + + _, err := CardsList("set123") + if err == nil { + t.Fatal("expected error for bad JSON payload") + } +} + +func TestCardsList_EmptyResult(t *testing.T) { + original := getCardData + defer func() { getCardData = original }() + + getCardData = func(url string) ([]byte, error) { + return []byte("[]"), nil + } + + model, err := CardsList("set123") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(model.PriceMap) != 0 || len(model.IllustratorMap) != 0 || len(model.ImageMap) != 0 { + t.Errorf("expected empty maps, got price:%d illus:%d image:%d", len(model.PriceMap), len(model.IllustratorMap), len(model.ImageMap)) + } + + if model.View() == "" { + t.Error("expected view to render with empty data") + } +} diff --git a/cmd/card/serieslist_test.go b/cmd/card/serieslist_test.go index e77926b..d339ed8 100644 --- a/cmd/card/serieslist_test.go +++ b/cmd/card/serieslist_test.go @@ -142,3 +142,26 @@ func TestSeriesModelView(t *testing.T) { t.Errorf("Expected non-empty view for choice, got empty string") } } + +func TestSeriesList(t *testing.T) { + model := SeriesList() + items := model.List.Items() + + // Check that list has 3 items + if items == nil { + t.Error("SeriesList() should create a list with items") + } + + if len(items) != 3 { + t.Errorf("Expected 3 items, got %d", len(items)) + } + + // Verify all three series are present + expectedSeries := []string{"Mega Evolution", "Scarlet & Violet", "Sword & Shield"} + for i, expected := range expectedSeries { + itemStr := string(items[i].(item)) + if itemStr != expected { + t.Errorf("Expected item %d to be '%s', got '%s'", i, expected, itemStr) + } + } +} From db9159f2452b09c525978d953d1940be1473dba8 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Mon, 15 Dec 2025 09:52:52 -0800 Subject: [PATCH 24/27] filtering out Pokeball and Masterball rarities --- .../pipelines/defs/extract/extract_pricing_data.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/card_data/pipelines/defs/extract/extract_pricing_data.py b/card_data/pipelines/defs/extract/extract_pricing_data.py index bf8699f..101b6b3 100644 --- a/card_data/pipelines/defs/extract/extract_pricing_data.py +++ b/card_data/pipelines/defs/extract/extract_pricing_data.py @@ -84,8 +84,6 @@ def extract_card_name(full_name: str) -> str: # Remove known variant types in parentheses # e.g., "(Secret)", "(Full Art)", "(Reverse Holofoil)", etc. variant_types = [ - "Poke Ball Pattern", - "Master Ball Pattern", "Full Art", "Secret", "Reverse Holofoil", @@ -96,7 +94,6 @@ def extract_card_name(full_name: str) -> str: name = name.replace(f" ({variant})", "") # Normalize accented characters (é → e, ñ → n, etc.) - # NFD decomposes characters into base + diacritics, then we filter out diacritics name = unicodedata.normalize("NFD", name) name = "".join(char for char in name if unicodedata.category(char) != "Mn") @@ -129,9 +126,14 @@ def pull_product_information(set_number: str) -> pl.DataFrame: if not is_card(card): continue + # Skip ball pattern variants (unique to Prismatic Evolutions) + card_name = card.get("name", "") + if "(Poke Ball Pattern)" in card_name or "(Master Ball Pattern)" in card_name: + continue + card_info = { "product_id": card["productId"], - "name": extract_card_name(card["name"]), + "name": extract_card_name(card_name), "card_number": get_card_number(card), "market_price": price_dict.get(card["productId"]), } From 3c2bc3e4acc2c84ae9b0d0b7f8b7dec2ae443bf0 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Mon, 15 Dec 2025 09:54:01 -0800 Subject: [PATCH 25/27] adding paths to ignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index c4fae95..1b8277f 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,5 @@ card_data/pipelines/poke_cli_dbt/.user.yml card_data/~/ card_data/storage/ +/.claude/ +CLAUDE.md From f47b950298e096ff1a15b1ce858971d4916955ff Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Mon, 15 Dec 2025 10:34:38 -0800 Subject: [PATCH 26/27] moving file to tests directory (#216) --- .../extract/testing.py => tests/extract_series_test.py} | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) rename card_data/pipelines/{defs/extract/testing.py => tests/extract_series_test.py} (87%) diff --git a/card_data/pipelines/defs/extract/testing.py b/card_data/pipelines/tests/extract_series_test.py similarity index 87% rename from card_data/pipelines/defs/extract/testing.py rename to card_data/pipelines/tests/extract_series_test.py index 92a47f4..3675d5b 100644 --- a/card_data/pipelines/defs/extract/testing.py +++ b/card_data/pipelines/tests/extract_series_test.py @@ -1,7 +1,12 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + import pytest import polars as pl import responses -from .extract_data import extract_series_data +from pipelines.defs.extract.extract_data import extract_series_data @pytest.fixture def mock_api_response(): From be66aa6316e85f6e1f40e469e038f3740de03c2f Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Mon, 15 Dec 2025 10:41:04 -0800 Subject: [PATCH 27/27] updating comments --- card_data/pipelines/tests/extract_series_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/card_data/pipelines/tests/extract_series_test.py b/card_data/pipelines/tests/extract_series_test.py index 3675d5b..c89303c 100644 --- a/card_data/pipelines/tests/extract_series_test.py +++ b/card_data/pipelines/tests/extract_series_test.py @@ -14,9 +14,9 @@ def mock_api_response(): return [ {"id": "sv", "name": "Scarlet & Violet", "logo": "https://example.com/sv.png"}, {"id": "swsh", "name": "Sword & Shield", "logo": "https://example.com/swsh.png"}, - {"id": "xy", "name": "XY", "logo": "https://example.com/xy.png"}, # Should be filtered out + {"id": "xy", "name": "XY", "logo": "https://example.com/xy.png"}, {"id": "me", "name": "McDonald's Collection", "logo": "https://example.com/me.png"}, - {"id": "sm", "name": "Sun & Moon", "logo": None}, # Should be filtered out + {"id": "sm", "name": "Sun & Moon", "logo": None}, ] @responses.activate