From a4cf7ee4b69a9c38817ef020e634c46f747e4f4e Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Thu, 19 Dec 2024 15:35:16 +0000 Subject: [PATCH 1/6] Ideal (not working) costs module --- src/muse/outputs/mca.py | 462 +++++++++++++++------------------------- 1 file changed, 171 insertions(+), 291 deletions(-) diff --git a/src/muse/outputs/mca.py b/src/muse/outputs/mca.py index fa38b2d3c..01ab30f07 100644 --- a/src/muse/outputs/mca.py +++ b/src/muse/outputs/mca.py @@ -30,7 +30,6 @@ def quantity( cast, ) -import numpy as np import pandas as pd import xarray as xr from mypy_extensions import KwArg @@ -38,7 +37,7 @@ def quantity( from muse.outputs.sector import market_quantity from muse.registration import registrator from muse.sectors import AbstractSector -from muse.timeslices import broadcast_timeslice, distribute_timeslice +from muse.sectors.preset_sector import PresetSector from muse.utilities import multiindex_to_coords OUTPUT_QUANTITY_SIGNATURE = Callable[ @@ -194,42 +193,29 @@ def capacity( def sector_capacity(sector: AbstractSector) -> pd.DataFrame: """Sector capacity with agent annotations.""" - capa_sector: list[xr.DataArray] = [] - agents = sorted(getattr(sector, "agents", []), key=attrgetter("name")) - for agent in agents: - capa_agent = agent.assets.capacity - capa_agent["agent"] = agent.name - capa_agent["type"] = agent.category - capa_agent["sector"] = getattr(sector, "name", "unnamed") - - if len(capa_agent) > 0 and len(capa_agent.technology.values) > 0: - if "dst_region" not in capa_agent.coords: - capa_agent["dst_region"] = agent.region - a = capa_agent.to_dataframe() - b = ( - a.groupby( - [ - "technology", - "dst_region", - "region", - "agent", - "sector", - "type", - "year", - "installed", - ] - ) - .sum() # ("asset") - .fillna(0) - ) - c = b.reset_index() - capa_sector.append(c) - if len(capa_sector) == 0: + if isinstance(sector, PresetSector): return pd.DataFrame() - capacity = pd.concat([u for u in capa_sector]) - capacity = capacity[capacity.capacity != 0] - return capacity + # Get data for the sector + data_sector: list[xr.DataArray] = [] + agents = sorted(getattr(sector, "agents"), key=attrgetter("name")) + + # Select data for the current year + current_year = agents[0].year + + # Get capacity data for each agent + for agent in agents: + assets = agent.assets.sel(year=current_year) + data_agent = assets.capacity + data_agent["agent"] = agent.name + data_agent["category"] = agent.category + data_agent["sector"] = getattr(sector, "name", "unnamed") + data_agent["year"] = current_year + data_agent = data_agent.to_dataframe("capacity") + data_sector.append(data_agent) + + output = pd.concat(data_sector, sort=True).reset_index() + return output def _aggregate_sectors( @@ -255,53 +241,39 @@ def sector_fuel_costs( sector: AbstractSector, market: xr.Dataset, year: int, **kwargs ) -> pd.DataFrame: """Sector fuel costs with agent annotations.""" - from muse.commodities import is_fuel - from muse.production import supply - from muse.quantities import consumption + from muse.costs import fuel_costs + + if isinstance(sector, PresetSector): + return pd.DataFrame() + # Get data for the sector data_sector: list[xr.DataArray] = [] - technologies = getattr(sector, "technologies", []) - agents = sorted(getattr(sector, "agents", []), key=attrgetter("name")) - - agent_market = market.copy(deep=True) - if len(technologies) > 0: - for a in agents: - agent_market["consumption"] = (market.consumption * a.quantity).sel( - year=year - ) - commodity = is_fuel(technologies.comm_usage) - - capacity = a.filter_input( - a.assets.capacity, - year=year, - ).fillna(0.0) - - production = supply( - agent_market, - capacity, - technologies, - ) - - prices = a.filter_input(market.prices, year=year) - fcons = consumption( - technologies=technologies, production=production, prices=prices - ) - - data_agent = (fcons * prices).sel(commodity=commodity) - data_agent["agent"] = a.name - data_agent["category"] = a.category - data_agent["sector"] = getattr(sector, "name", "unnamed") - data_agent["year"] = year - data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe( - "fuel_consumption_costs" - ) - if not data_agent.empty: - data_sector.append(data_agent) - if len(data_sector) > 0: - output = pd.concat(data_sector, sort=True).reset_index() - else: - output = pd.DataFrame() + technologies = getattr(sector, "technologies") + agents = sorted(getattr(sector, "agents"), key=attrgetter("name")) + # Select data for the current year + current_year = agents[0].year + _technologies = technologies.sel(year=current_year) + _market = market.sel(year=current_year, commodity=_technologies.commodity) + + # Calculate fuel costs + for agent in agents: + assets = agent.assets.sel(year=current_year) + data_agent = fuel_costs( + technologies=_technologies, + prices=_market.prices, + consumption=assets.consumption, + ) + data_agent["agent"] = agent.name + data_agent["category"] = agent.category + data_agent["sector"] = getattr(sector, "name", "unnamed") + data_agent["year"] = current_year + data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe( + "fuel_consumption_costs" + ) + data_sector.append(data_agent) + + output = pd.concat(data_sector, sort=True).reset_index() return output @@ -317,33 +289,34 @@ def sector_capital_costs( sector: AbstractSector, market: xr.Dataset, year: int, **kwargs ) -> pd.DataFrame: """Sector capital costs with agent annotations.""" + from muse.costs import capital_costs + + if isinstance(sector, PresetSector): + return pd.DataFrame() + + # Get data for the sector data_sector: list[xr.DataArray] = [] - technologies = getattr(sector, "technologies", []) - agents = sorted(getattr(sector, "agents", []), key=attrgetter("name")) - - if len(technologies) > 0: - for a in agents: - capacity = a.filter_input(a.assets.capacity, year=year).fillna(0.0) - data = a.filter_input( - technologies[["cap_par", "cap_exp"]], - year=year, - technology=capacity.technology, - ) - data_agent = distribute_timeslice(data.cap_par * (capacity**data.cap_exp)) - data_agent["agent"] = a.name - data_agent["category"] = a.category - data_agent["sector"] = getattr(sector, "name", "unnamed") - data_agent["year"] = year - data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe( - "capital_costs" - ) - if not data_agent.empty: - data_sector.append(data_agent) - - if len(data_sector) > 0: - output = pd.concat(data_sector, sort=True).reset_index() - else: - output = pd.DataFrame() + technologies = getattr(sector, "technologies") + agents = sorted(getattr(sector, "agents"), key=attrgetter("name")) + + # Select data for the current year + current_year = agents[0].year + _technologies = technologies.sel(year=current_year) + + # Calculate capital costs + for agent in agents: + assets = agent.assets.sel(year=current_year) + data_agent = capital_costs( + technologies=_technologies, capacity=assets.capacity, method="annual" + ) + data_agent["agent"] = agent.name + data_agent["category"] = agent.category + data_agent["sector"] = getattr(sector, "name", "unnamed") + data_agent["year"] = current_year + data_agent = data_agent.to_dataframe("capital_costs") + data_sector.append(data_agent) + + output = pd.concat(data_sector, sort=True).reset_index() return output @@ -359,55 +332,37 @@ def sector_emission_costs( sector: AbstractSector, market: xr.Dataset, year: int, **kwargs ) -> pd.DataFrame: """Sector emission costs with agent annotations.""" - from muse.commodities import is_enduse, is_pollutant - from muse.production import supply + from muse.costs import environmental_costs + + if isinstance(sector, PresetSector): + return pd.DataFrame() + # Get data for the sector data_sector: list[xr.DataArray] = [] - technologies = getattr(sector, "technologies", []) - agents = sorted(getattr(sector, "agents", []), key=attrgetter("name")) - - agent_market = market.copy(deep=True) - if len(technologies) > 0: - for a in agents: - agent_market["consumption"] = (market.consumption * a.quantity).sel( - year=year - ) - - capacity = a.filter_input(a.assets.capacity, year=year).fillna(0.0) - allemissions = a.filter_input( - technologies.fixed_outputs, - commodity=is_pollutant(technologies.comm_usage), - technology=capacity.technology, - year=year, - ) - envs = is_pollutant(technologies.comm_usage) - enduses = is_enduse(technologies.comm_usage) - i = (np.where(envs))[0][0] - red_envs = envs[i].commodity.values - prices = a.filter_input(market.prices, year=year, commodity=red_envs) - production = supply( - agent_market, - capacity, - technologies, - ) - - total = production.sel(commodity=enduses).sum("commodity") - data_agent = total * (allemissions * prices).sum("commodity") - data_agent["agent"] = a.name - data_agent["category"] = a.category - data_agent["sector"] = getattr(sector, "name", "unnamed") - data_agent["year"] = year - data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe( - "emission_costs" - ) - if not data_agent.empty: - data_sector.append(data_agent) - - if len(data_sector) > 0: - output = pd.concat(data_sector, sort=True).reset_index() - else: - output = pd.DataFrame() + technologies = getattr(sector, "technologies") + agents = sorted(getattr(sector, "agents"), key=attrgetter("name")) + + # Select data for the current year + current_year = agents[0].year + _technologies = technologies.sel(year=current_year) + _market = market.sel(year=current_year, commodity=technologies.commodity) + # Calculate emission costs + for agent in agents: + assets = agent.assets.sel(year=current_year) + data_agent = environmental_costs( + technologies=_technologies, prices=_market.prices, production=assets.supply + ) + data_agent["agent"] = agent.name + data_agent["category"] = agent.category + data_agent["sector"] = getattr(sector, "name", "unnamed") + data_agent["year"] = current_year + data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe( + "emission_costs" + ) + data_sector.append(data_agent) + + output = pd.concat(data_sector, sort=True).reset_index() return output @@ -423,78 +378,40 @@ def sector_lcoe( sector: AbstractSector, market: xr.Dataset, year: int, **kwargs ) -> pd.DataFrame: """Levelized cost of energy () of technologies over their lifetime.""" - from muse.costs import levelized_cost_of_energy as LCOE - from muse.quantities import capacity_to_service_demand, consumption + from muse.costs import levelized_cost_of_energy - market = market.copy(deep=True) + if isinstance(sector, PresetSector): + return pd.DataFrame() - # Filtering of the inputs + # Get data for the sector data_sector: list[xr.DataArray] = [] - technologies = getattr(sector, "technologies", []) - agents = sorted(getattr(sector, "agents", []), key=attrgetter("name")) - retro = [a for a in agents if a.category == "retrofit"] - new = [a for a in agents if a.category == "newcapa"] - agents = retro if len(retro) > 0 else new - if len(technologies) > 0: - for agent in agents: - agent_market = market.sel(year=agent.year) - agent_market["consumption"] = agent_market.consumption * agent.quantity - enduses = [ - i.strip() - for entry in technologies.enduse.values - for i in entry.split(",") - ] - # temporary hack to allow comma separated list in input file - included = [i for i in agent_market["commodity"].values if i in enduses] - excluded = [ - i for i in agent_market["commodity"].values if i not in included - ] - agent_market.loc[dict(commodity=excluded)] = 0 - agent_market["prices"] = agent.filter_input( - market["prices"], year=agent.year - ) - - techs = agent.filter_input( - technologies, - year=agent.year, - ) - prices = agent_market["prices"].sel(commodity=techs.commodity) - demand = agent_market.consumption.sel(commodity=included) - capacity = agent.filter_input(capacity_to_service_demand(demand, techs)) - production = ( - broadcast_timeslice(capacity) - * distribute_timeslice(techs.fixed_outputs) - * broadcast_timeslice(techs.utilization_factor) - ) - consump = consumption( - technologies=techs, prices=prices, production=production - ) - - result = LCOE( - technologies=techs, - prices=prices, - capacity=capacity, - production=production, - consumption=consump, - method="lifetime", - ) - - data_agent = result - data_agent["agent"] = agent.name - data_agent["category"] = agent.category - data_agent["sector"] = getattr(sector, "name", "unnamed") - data_agent["year"] = agent.year - data_agent = data_agent.fillna(0) - data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe( - "LCOE" - ) - if not data_agent.empty: - data_sector.append(data_agent) - - if len(data_sector) > 0: - output = pd.concat(data_sector, sort=True).reset_index() - else: - output = pd.DataFrame() + technologies = getattr(sector, "technologies") + agents = sorted(getattr(sector, "agents"), key=attrgetter("name")) + + # Select data for the current year + current_year = agents[0].year + _technologies = technologies.sel(year=current_year) + _market = market.sel(year=current_year, commodity=technologies.commodity) + + # Calculate LCOE + for agent in agents: + assets = agent.assets.sel(year=current_year) + data_agent = levelized_cost_of_energy( + technologies=_technologies, + prices=_market.prices, + capacity=assets.capacity, + production=assets.supply, + consumption=assets.consumption, + method="annual", + ) + data_agent["agent"] = agent.name + data_agent["category"] = agent.category + data_agent["sector"] = getattr(sector, "name", "unnamed") + data_agent["year"] = current_year + data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe("lcoe") + data_sector.append(data_agent) + + output = pd.concat(data_sector, sort=True).reset_index() return output @@ -506,77 +423,40 @@ def metric_eac( return _aggregate_sectors(sectors, market, year, op=sector_eac) -def sector_eac( - sector: AbstractSector, market: xr.Dataset, year: int, **kwargs -) -> pd.DataFrame: - """Net Present Value of technologies over their lifetime.""" - from muse.costs import equivalent_annual_cost as EAC - from muse.quantities import capacity_to_service_demand, consumption +def sector_eac(sector: AbstractSector, market: xr.Dataset, **kwargs) -> pd.DataFrame: + """Equivalent Annual Cost of technologies over their lifetime.""" + from muse.costs import equivalent_annual_cost - market = market.copy(deep=True) + if isinstance(sector, PresetSector): + return pd.DataFrame() - # Filtering of the inputs + # Get data for the sector data_sector: list[xr.DataArray] = [] - technologies = getattr(sector, "technologies", []) - agents = sorted(getattr(sector, "agents", []), key=attrgetter("name")) - retro = [a for a in agents if a.category == "retrofit"] - new = [a for a in agents if a.category == "newcapa"] - agents = retro if len(retro) > 0 else new - if len(technologies) > 0: - for agent in agents: - agent_market = market.sel(year=agent.year) - agent_market["consumption"] = agent_market.consumption * agent.quantity - enduses = [ - i.strip() - for entry in technologies.enduse.values - for i in entry.split(",") - ] - # temporary hack to allow comma separated list in input file - included = [i for i in agent_market["commodity"].values if i in enduses] - excluded = [ - i for i in agent_market["commodity"].values if i not in included - ] - agent_market.loc[dict(commodity=excluded)] = 0 - agent_market["prices"] = agent.filter_input( - market["prices"], year=agent.year - ) - - techs = agent.filter_input( - technologies, - year=agent.year, - ) - prices = agent_market["prices"].sel(commodity=techs.commodity) - demand = agent_market.consumption.sel(commodity=included) - capacity = agent.filter_input(capacity_to_service_demand(demand, techs)) - production = ( - broadcast_timeslice(capacity) - * distribute_timeslice(techs.fixed_outputs) - * broadcast_timeslice(techs.utilization_factor) - ) - consump = consumption( - technologies=techs, prices=prices, production=production - ) - - result = EAC( - technologies=techs, - prices=prices, - capacity=capacity, - production=production, - consumption=consump, - ) - - data_agent = result - data_agent["agent"] = agent.name - data_agent["category"] = agent.category - data_agent["sector"] = getattr(sector, "name", "unnamed") - data_agent["year"] = agent.year - data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe( - "capital_costs" - ) - if not data_agent.empty: - data_sector.append(data_agent) - if len(data_sector) > 0: - output = pd.concat(data_sector, sort=True).reset_index() - else: - output = pd.DataFrame() + technologies = getattr(sector, "technologies") + agents = sorted(getattr(sector, "agents"), key=attrgetter("name")) + + # Select data for the current year + current_year = agents[0].year + _technologies = technologies.sel(year=current_year) + _market = market.sel(year=current_year, commodity=technologies.commodity) + + # Calculate EAC + for agent in agents: + assets = agent.assets.sel(year=current_year) + data_agent = equivalent_annual_cost( + technologies=_technologies, + prices=_market.prices, + capacity=assets.capacity, + production=assets.supply, + consumption=assets.consumption, + method="annual", + ) + data_agent["agent"] = agent.name + data_agent["category"] = agent.category + data_agent["sector"] = getattr(sector, "name", "unnamed") + data_agent["year"] = current_year + data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe("lcoe") + data_sector.append(data_agent) + + output = pd.concat(data_sector, sort=True).reset_index() return output From 0973b4652ca2e964ec29786ed1806dfc0fd8672e Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Mon, 2 Jun 2025 15:27:52 +0100 Subject: [PATCH 2/6] Save supply and consumption attributes of assets --- src/muse/agents/agent.py | 6 ------ src/muse/sectors/sector.py | 19 +++++++++++++++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/muse/agents/agent.py b/src/muse/agents/agent.py index eddc6600e..9f255126c 100644 --- a/src/muse/agents/agent.py +++ b/src/muse/agents/agent.py @@ -481,10 +481,4 @@ def retirement_profile( # Apply the retirement profile to the investments new_assets = (investments * profile).rename(replacement="asset") new_assets["installed"] = "asset", [investment_year] * len(new_assets.asset) - - # The new assets have picked up quite a few coordinates along the way. - # we try and keep only those that were there originally. - if set(new_assets.dims) != set(self.assets.dims): - new, old = new_assets.dims, self.assets.dims - raise RuntimeError(f"Asset dimensions do not match: {new} vs {old}") return new_assets diff --git a/src/muse/sectors/sector.py b/src/muse/sectors/sector.py index 00d2532a1..7e1ae2602 100644 --- a/src/muse/sectors/sector.py +++ b/src/muse/sectors/sector.py @@ -281,10 +281,19 @@ def market_variables(self, market: xr.Dataset, technologies: xr.Dataset) -> Any: from muse.commodities import is_pollutant from muse.costs import levelized_cost_of_energy, supply_cost from muse.quantities import consumption - from muse.utilities import broadcast_over_assets, interpolate_capacity + from muse.utilities import ( + agent_concatenation, + broadcast_over_assets, + interpolate_capacity, + ) years = market.year.values - capacity = interpolate_capacity(self.capacity, year=years) + + # Create a concatenated view of all agents' assets with agent mapping + agent_assets = agent_concatenation( + {agent.uuid: agent.assets for agent in self.agents} + ) + capacity = interpolate_capacity(agent_assets.capacity, year=years) # Select technology data for each asset # Each asset uses the technology data from the year it was installed @@ -326,6 +335,12 @@ def market_variables(self, market: xr.Dataset, technologies: xr.Dataset) -> Any: asset_dim="asset", ) + # Distribute supply and consumption back to agents using the agent coordinate + for agent in self.agents: + agent_mask = agent_assets.agent == agent.uuid + agent.assets["supply"] = supply.sel(asset=agent_mask) + agent.assets["consumption"] = consume.sel(asset=agent_mask) + return supply, consume, costs def convert_to_sector_timeslicing(self, market: xr.Dataset) -> xr.Dataset: From 5f5412d8bda3c7601aa45d4c778764357857dba9 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Mon, 2 Jun 2025 16:15:23 +0100 Subject: [PATCH 3/6] First working version --- src/muse/agents/agent.py | 6 +++ src/muse/outputs/mca.py | 107 +++++++++++++++++-------------------- src/muse/sectors/sector.py | 4 +- 3 files changed, 56 insertions(+), 61 deletions(-) diff --git a/src/muse/agents/agent.py b/src/muse/agents/agent.py index f12cbdb4e..d5f085494 100644 --- a/src/muse/agents/agent.py +++ b/src/muse/agents/agent.py @@ -489,4 +489,10 @@ def retirement_profile( # Apply the retirement profile to the investments new_assets = (investments * profile).rename(replacement="asset") new_assets["installed"] = "asset", [investment_year] * len(new_assets.asset) + + # The new assets have picked up quite a few coordinates along the way. + # we try and keep only those that were there originally. + if set(new_assets.dims) != set(self.assets.dims): + new, old = new_assets.dims, self.assets.dims + raise RuntimeError(f"Asset dimensions do not match: {new} vs {old}") return new_assets diff --git a/src/muse/outputs/mca.py b/src/muse/outputs/mca.py index 01ab30f07..0be689fb5 100644 --- a/src/muse/outputs/mca.py +++ b/src/muse/outputs/mca.py @@ -188,10 +188,10 @@ def capacity( market: xr.Dataset, sectors: list[AbstractSector], year: int, **kwargs ) -> pd.DataFrame: """Current capacity across all sectors.""" - return _aggregate_sectors(sectors, op=sector_capacity) + return _aggregate_sectors(sectors, year, op=sector_capacity) -def sector_capacity(sector: AbstractSector) -> pd.DataFrame: +def sector_capacity(sector: AbstractSector, year: int) -> pd.DataFrame: """Sector capacity with agent annotations.""" if isinstance(sector, PresetSector): return pd.DataFrame() @@ -200,17 +200,13 @@ def sector_capacity(sector: AbstractSector) -> pd.DataFrame: data_sector: list[xr.DataArray] = [] agents = sorted(getattr(sector, "agents"), key=attrgetter("name")) - # Select data for the current year - current_year = agents[0].year - # Get capacity data for each agent for agent in agents: - assets = agent.assets.sel(year=current_year) - data_agent = assets.capacity + data_agent = agent.assets.capacity.sel(year=year) data_agent["agent"] = agent.name data_agent["category"] = agent.category data_agent["sector"] = getattr(sector, "name", "unnamed") - data_agent["year"] = current_year + data_agent["year"] = year data_agent = data_agent.to_dataframe("capacity") data_sector.append(data_agent) @@ -242,6 +238,7 @@ def sector_fuel_costs( ) -> pd.DataFrame: """Sector fuel costs with agent annotations.""" from muse.costs import fuel_costs + from muse.utilities import broadcast_over_assets if isinstance(sector, PresetSector): return pd.DataFrame() @@ -251,23 +248,20 @@ def sector_fuel_costs( technologies = getattr(sector, "technologies") agents = sorted(getattr(sector, "agents"), key=attrgetter("name")) - # Select data for the current year - current_year = agents[0].year - _technologies = technologies.sel(year=current_year) - _market = market.sel(year=current_year, commodity=_technologies.commodity) - # Calculate fuel costs + _market = market.sel(year=year, commodity=technologies.commodity) for agent in agents: - assets = agent.assets.sel(year=current_year) data_agent = fuel_costs( - technologies=_technologies, - prices=_market.prices, - consumption=assets.consumption, + technologies=broadcast_over_assets(technologies, agent.assets), + prices=broadcast_over_assets( + _market.prices, agent.assets, installed_as_year=False + ), + consumption=agent.consumption.sel(year=year), ) data_agent["agent"] = agent.name data_agent["category"] = agent.category data_agent["sector"] = getattr(sector, "name", "unnamed") - data_agent["year"] = current_year + data_agent["year"] = year data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe( "fuel_consumption_costs" ) @@ -290,6 +284,7 @@ def sector_capital_costs( ) -> pd.DataFrame: """Sector capital costs with agent annotations.""" from muse.costs import capital_costs + from muse.utilities import broadcast_over_assets if isinstance(sector, PresetSector): return pd.DataFrame() @@ -299,20 +294,17 @@ def sector_capital_costs( technologies = getattr(sector, "technologies") agents = sorted(getattr(sector, "agents"), key=attrgetter("name")) - # Select data for the current year - current_year = agents[0].year - _technologies = technologies.sel(year=current_year) - # Calculate capital costs for agent in agents: - assets = agent.assets.sel(year=current_year) data_agent = capital_costs( - technologies=_technologies, capacity=assets.capacity, method="annual" + technologies=broadcast_over_assets(technologies, agent.assets), + capacity=agent.assets.capacity.sel(year=year), + method="annual", ) data_agent["agent"] = agent.name data_agent["category"] = agent.category data_agent["sector"] = getattr(sector, "name", "unnamed") - data_agent["year"] = current_year + data_agent["year"] = year data_agent = data_agent.to_dataframe("capital_costs") data_sector.append(data_agent) @@ -333,6 +325,7 @@ def sector_emission_costs( ) -> pd.DataFrame: """Sector emission costs with agent annotations.""" from muse.costs import environmental_costs + from muse.utilities import broadcast_over_assets if isinstance(sector, PresetSector): return pd.DataFrame() @@ -342,21 +335,20 @@ def sector_emission_costs( technologies = getattr(sector, "technologies") agents = sorted(getattr(sector, "agents"), key=attrgetter("name")) - # Select data for the current year - current_year = agents[0].year - _technologies = technologies.sel(year=current_year) - _market = market.sel(year=current_year, commodity=technologies.commodity) - # Calculate emission costs + _market = market.sel(year=year, commodity=technologies.commodity) for agent in agents: - assets = agent.assets.sel(year=current_year) data_agent = environmental_costs( - technologies=_technologies, prices=_market.prices, production=assets.supply + technologies=broadcast_over_assets(technologies, agent.assets), + prices=broadcast_over_assets( + _market.prices, agent.assets, installed_as_year=False + ), + production=agent.supply.sel(year=year), ) data_agent["agent"] = agent.name data_agent["category"] = agent.category data_agent["sector"] = getattr(sector, "name", "unnamed") - data_agent["year"] = current_year + data_agent["year"] = year data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe( "emission_costs" ) @@ -379,6 +371,7 @@ def sector_lcoe( ) -> pd.DataFrame: """Levelized cost of energy () of technologies over their lifetime.""" from muse.costs import levelized_cost_of_energy + from muse.utilities import broadcast_over_assets if isinstance(sector, PresetSector): return pd.DataFrame() @@ -388,26 +381,23 @@ def sector_lcoe( technologies = getattr(sector, "technologies") agents = sorted(getattr(sector, "agents"), key=attrgetter("name")) - # Select data for the current year - current_year = agents[0].year - _technologies = technologies.sel(year=current_year) - _market = market.sel(year=current_year, commodity=technologies.commodity) - # Calculate LCOE + _market = market.sel(year=year, commodity=technologies.commodity) for agent in agents: - assets = agent.assets.sel(year=current_year) data_agent = levelized_cost_of_energy( - technologies=_technologies, - prices=_market.prices, - capacity=assets.capacity, - production=assets.supply, - consumption=assets.consumption, + technologies=broadcast_over_assets(technologies, agent.assets), + prices=broadcast_over_assets( + _market.prices, agent.assets, installed_as_year=False + ), + capacity=agent.assets.capacity.sel(year=year), + production=agent.supply.sel(year=year), + consumption=agent.consumption.sel(year=year), method="annual", ) data_agent["agent"] = agent.name data_agent["category"] = agent.category data_agent["sector"] = getattr(sector, "name", "unnamed") - data_agent["year"] = current_year + data_agent["year"] = year data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe("lcoe") data_sector.append(data_agent) @@ -423,9 +413,12 @@ def metric_eac( return _aggregate_sectors(sectors, market, year, op=sector_eac) -def sector_eac(sector: AbstractSector, market: xr.Dataset, **kwargs) -> pd.DataFrame: +def sector_eac( + sector: AbstractSector, market: xr.Dataset, year: int, **kwargs +) -> pd.DataFrame: """Equivalent Annual Cost of technologies over their lifetime.""" from muse.costs import equivalent_annual_cost + from muse.utilities import broadcast_over_assets if isinstance(sector, PresetSector): return pd.DataFrame() @@ -435,26 +428,22 @@ def sector_eac(sector: AbstractSector, market: xr.Dataset, **kwargs) -> pd.DataF technologies = getattr(sector, "technologies") agents = sorted(getattr(sector, "agents"), key=attrgetter("name")) - # Select data for the current year - current_year = agents[0].year - _technologies = technologies.sel(year=current_year) - _market = market.sel(year=current_year, commodity=technologies.commodity) - # Calculate EAC + _market = market.sel(year=year, commodity=technologies.commodity) for agent in agents: - assets = agent.assets.sel(year=current_year) data_agent = equivalent_annual_cost( - technologies=_technologies, - prices=_market.prices, - capacity=assets.capacity, - production=assets.supply, - consumption=assets.consumption, - method="annual", + technologies=broadcast_over_assets(technologies, agent.assets), + prices=broadcast_over_assets( + _market.prices, agent.assets, installed_as_year=False + ), + capacity=agent.assets.capacity.sel(year=year), + production=agent.supply.sel(year=year), + consumption=agent.consumption.sel(year=year), ) data_agent["agent"] = agent.name data_agent["category"] = agent.category data_agent["sector"] = getattr(sector, "name", "unnamed") - data_agent["year"] = current_year + data_agent["year"] = year data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe("lcoe") data_sector.append(data_agent) diff --git a/src/muse/sectors/sector.py b/src/muse/sectors/sector.py index 27d840ec2..9e369f904 100644 --- a/src/muse/sectors/sector.py +++ b/src/muse/sectors/sector.py @@ -331,8 +331,8 @@ def market_variables(self, market: xr.Dataset, technologies: xr.Dataset) -> Any: # Distribute supply and consumption back to agents using the agent coordinate for agent in self.agents: agent_mask = agent_assets.agent == agent.uuid - agent.assets["supply"] = supply.sel(asset=agent_mask) - agent.assets["consumption"] = consume.sel(asset=agent_mask) + agent.supply = supply.sel(asset=agent_mask) + agent.consumption = consume.sel(asset=agent_mask) return supply, consume, costs From 445777c93769d2b45d449c765b3018e013e22259 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Mon, 2 Jun 2025 16:20:47 +0100 Subject: [PATCH 4/6] Delete outdated tests --- tests/test_aggregoutput.py | 120 ------------------------------------- 1 file changed, 120 deletions(-) delete mode 100644 tests/test_aggregoutput.py diff --git a/tests/test_aggregoutput.py b/tests/test_aggregoutput.py deleted file mode 100644 index f11e8e74b..000000000 --- a/tests/test_aggregoutput.py +++ /dev/null @@ -1,120 +0,0 @@ -from muse import examples -from muse.outputs.mca import sector_capacity - - -def test_aggregate_sector(): - """Test for aggregate_sector function. - - Check column titles, number of agents/region/technologies and assets capacities. - """ - from pandas import DataFrame, concat - - mca = examples.model("multiple_agents", test=True) - year = [2020, 2025] - sector_list = [sector for sector in mca.sectors if "preset" not in sector.name] - agent_list = [list(a.agents) for a in sector_list] - alldata = sector_capacity(sector_list[0]) - - columns = ["region", "agent", "type", "sector", "capacity"] - - frame = DataFrame() - for ai in agent_list[0]: - for y in year: - if y in ai.assets.year: - if ai.assets.capacity.sel(year=y).values > 0.0: - data = DataFrame( - { - "region": ai.region, - "agent": ai.name, - "type": ai.category, - "sector": sector_list[0].name, - "capacity": ai.assets.capacity.sel(year=y).values[0], - }, - index=[(y, ai.assets.technology.values[0])], - ) - frame = concat([frame, data]) - - assert (frame[columns].values == alldata[columns].values).all() - - -def test_aggregate_sectors(): - """Test for aggregate_sectors function.""" - from pandas import DataFrame, concat - - from muse.outputs.mca import _aggregate_sectors - - mca = examples.model("multiple_agents", test=True) - year = [2020, 2025, 2030] - sector_list = [sector for sector in mca.sectors if "preset" not in sector.name] - agent_list = [list(a.agents) for a in sector_list] - alldata = _aggregate_sectors(mca.sectors, op=sector_capacity) - - columns = ["region", "agent", "type", "sector", "capacity"] - - frame = DataFrame() - for a, ai in enumerate(agent_list): - for ii in range(0, len(ai)): - for y in year: - if y in ai[ii].assets.year: - if ai[ii].assets.capacity.sel(year=y).values > 0.0: - data = DataFrame( - { - "region": ai[ii].region, - "agent": ai[ii].name, - "type": ai[ii].category, - "sector": sector_list[a].name, - "capacity": ai[ii] - .assets.capacity.sel(year=y) - .values[0], - }, - index=[(y, ai[ii].assets.technology.values[0])], - ) - frame = concat([frame, data]) - - assert (frame[columns].values == alldata[columns].values).all() - - -def test_aggregate_sector_manyregions(): - """Test for aggregate_sector function with two regions. - - Check column titles, number of agents/region/technologies and assets capacities. - """ - from pandas import DataFrame, concat - - from muse.outputs.mca import _aggregate_sectors - - mca = examples.model("multiple_agents", test=True) - residential = next(sector for sector in mca.sectors if sector.name == "residential") - agents = list(residential.agents) - agents[0].assets["region"] = "BELARUS" - agents[1].assets["region"] = "BELARUS" - agents[0].region = "BELARUS" - agents[1].region = "BELARUS" - year = [2020, 2025, 2030] - sector_list = [sector for sector in mca.sectors if "preset" not in sector.name] - agent_list = [list(a.agents) for a in sector_list] - alldata = _aggregate_sectors(mca.sectors, op=sector_capacity) - - columns = ["region", "agent", "type", "sector", "capacity"] - - frame = DataFrame() - for a, ai in enumerate(agent_list): - for ii in range(0, len(ai)): - for y in year: - if y in ai[ii].assets.year: - if ai[ii].assets.capacity.sel(year=y).values > 0.0: - data = DataFrame( - { - "region": ai[ii].region, - "agent": ai[ii].name, - "type": ai[ii].category, - "sector": sector_list[a].name, - "capacity": ai[ii] - .assets.capacity.sel(year=y) - .values[0], - }, - index=[(y, ai[ii].assets.technology.values[0])], - ) - frame = concat([frame, data]) - - assert (frame[columns].values == alldata[columns].values).all() From 9fd825b069317266ea85cd7b1f598cdbaf95707e Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Fri, 6 Jun 2025 11:20:26 +0100 Subject: [PATCH 5/6] Add tests (not yet working) --- src/muse/outputs/mca.py | 2 +- tests/test_mca_outputs.py | 209 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 tests/test_mca_outputs.py diff --git a/src/muse/outputs/mca.py b/src/muse/outputs/mca.py index 0be689fb5..5986df92d 100644 --- a/src/muse/outputs/mca.py +++ b/src/muse/outputs/mca.py @@ -444,7 +444,7 @@ def sector_eac( data_agent["category"] = agent.category data_agent["sector"] = getattr(sector, "name", "unnamed") data_agent["year"] = year - data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe("lcoe") + data_agent = multiindex_to_coords(data_agent, "timeslice").to_dataframe("eac") data_sector.append(data_agent) output = pd.concat(data_sector, sort=True).reset_index() diff --git a/tests/test_mca_outputs.py b/tests/test_mca_outputs.py new file mode 100644 index 000000000..3dbe849a7 --- /dev/null +++ b/tests/test_mca_outputs.py @@ -0,0 +1,209 @@ +"""Tests for MCA output quantities.""" + +import pandas as pd +from pytest import fixture + +from muse.outputs.mca import ( + capacity, + consumption, + metric_capital_costs, + metric_eac, + metric_emission_costs, + metric_fuel_costs, + metric_lcoe, + prices, + supply, +) + + +@fixture +def mock_sectors(model) -> list: + """Create test sectors using MUSE's examples module.""" + from muse import examples + + return [examples.sector("residential", model=model)] + + +def test_consumption(market, mock_sectors): + """Test consumption output quantity.""" + result = consumption(market, mock_sectors, 2010) + assert isinstance(result, pd.DataFrame) + assert not result.empty + # Market quantities include timeslice-related dimensions + expected_cols = { + "region", + "commodity", + "year", + "month", + "day", + "hour", + "timeslice", + "consumption", + } + assert set(result.columns) == expected_cols + + +def test_supply(market, mock_sectors): + """Test supply output quantity.""" + result = supply(market, mock_sectors, 2010) + assert isinstance(result, pd.DataFrame) + assert not result.empty + # Market quantities include timeslice-related dimensions + expected_cols = { + "region", + "commodity", + "year", + "month", + "day", + "hour", + "timeslice", + "supply", + } + assert set(result.columns) == expected_cols + + +def test_prices(market, mock_sectors): + """Test prices output quantity.""" + result = prices(market, mock_sectors, 2010) + assert isinstance(result, pd.DataFrame) + assert not result.empty + # Market quantities include timeslice-related dimensions + expected_cols = { + "region", + "commodity", + "year", + "month", + "day", + "hour", + "timeslice", + "prices", + } + assert set(result.columns) == expected_cols + + +def test_capacity(market, mock_sectors): + """Test capacity output quantity.""" + result = capacity(market, mock_sectors, 2010) + assert isinstance(result, pd.DataFrame) + assert not result.empty + expected_cols = { + "technology", + "agent", + "category", + "sector", + "year", + "capacity", + } + assert set(result.columns) == expected_cols + + +def test_fuel_costs(market, mock_sectors): + """Test fuel costs output quantity.""" + result = metric_fuel_costs(market, mock_sectors, 2010) + assert isinstance(result, pd.DataFrame) + assert not result.empty + # Cost quantities include timeslice-related dimensions and region + expected_cols = { + "technology", + "agent", + "category", + "sector", + "year", + "region", + "month", + "day", + "hour", + "timeslice", + "fuel_consumption_costs", + } + assert set(result.columns) == expected_cols + + +def test_capital_costs(market, mock_sectors): + """Test capital costs output quantity.""" + result = metric_capital_costs(market, mock_sectors, 2010) + assert isinstance(result, pd.DataFrame) + assert not result.empty + # Capital costs include technology attributes + expected_cols = { + "technology", + "agent", + "category", + "sector", + "year", + "tech_type", + "fuel", + "capital_costs", + "region", + } + assert set(result.columns) == expected_cols + + +def test_emission_costs(market, mock_sectors): + """Test emission costs output quantity.""" + result = metric_emission_costs(market, mock_sectors, 2010) + assert isinstance(result, pd.DataFrame) + assert not result.empty + # Cost quantities include timeslice-related dimensions and region + expected_cols = { + "technology", + "agent", + "category", + "sector", + "year", + "region", + "month", + "day", + "hour", + "timeslice", + "emission_costs", + } + assert set(result.columns) == expected_cols + + +def test_lcoe(market, mock_sectors): + """Test LCOE output quantity.""" + result = metric_lcoe(market, mock_sectors, 2010) + assert isinstance(result, pd.DataFrame) + assert not result.empty + # Cost quantities include timeslice-related dimensions and technology attributes + expected_cols = { + "technology", + "agent", + "category", + "sector", + "year", + "region", + "tech_type", + "fuel", + "month", + "day", + "hour", + "timeslice", + "lcoe", + } + assert set(result.columns) == expected_cols + + +def test_eac(market, mock_sectors): + """Test EAC output quantity.""" + result = metric_eac(market, mock_sectors, 2010) + assert isinstance(result, pd.DataFrame) + assert not result.empty + # Cost quantities include timeslice-related dimensions and technology attributes + expected_cols = { + "technology", + "agent", + "category", + "sector", + "year", + "region", + "tech_type", + "fuel", + "month", + "day", + "hour", + "timeslice", + "eac", + } + assert set(result.columns) == expected_cols From b0d63c73a422dc6f13438662416143ca3727b4a2 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Tue, 10 Jun 2025 16:28:38 +0100 Subject: [PATCH 6/6] Some progress on tests, but still not working --- tests/test_mca_outputs.py | 112 ++++++++++++++++++++++++-------------- 1 file changed, 70 insertions(+), 42 deletions(-) diff --git a/tests/test_mca_outputs.py b/tests/test_mca_outputs.py index 3dbe849a7..4a02c2f5e 100644 --- a/tests/test_mca_outputs.py +++ b/tests/test_mca_outputs.py @@ -1,8 +1,10 @@ """Tests for MCA output quantities.""" import pandas as pd +import xarray as xr from pytest import fixture +from muse import examples from muse.outputs.mca import ( capacity, consumption, @@ -14,22 +16,58 @@ prices, supply, ) +from muse.utilities import broadcast_over_assets +YEAR = 2020 -@fixture -def mock_sectors(model) -> list: - """Create test sectors using MUSE's examples module.""" - from muse import examples - return [examples.sector("residential", model=model)] +@fixture +def market() -> xr.Dataset: + """Create a test market.""" + return examples.residential_market(model="default") -def test_consumption(market, mock_sectors): +@fixture +def sectors(market) -> list: + """Create test sectors using MUSE's examples module.""" + residential_sector = examples.sector("residential", model="default") + agent = next(residential_sector.agents) + technologies = residential_sector.technologies + tech_data = broadcast_over_assets(technologies, agent.assets) + + # Make up supply data + supply_data = xr.DataArray( + data=1.0, + dims=["timeslice", "commodity", "year", "asset"], + coords={ + "timeslice": market.timeslice, + "commodity": tech_data.commodity, + "year": market.year, + "asset": agent.assets.asset, + }, + ) + agent.supply = supply_data + + # Make up consumption data + consumption_data = xr.DataArray( + data=1.0, + dims=["timeslice", "commodity", "year", "asset"], + coords={ + "timeslice": market.timeslice, + "commodity": tech_data.commodity, + "year": market.year, + "asset": agent.assets.asset, + }, + ) + agent.consumption = consumption_data + + return [residential_sector] + + +def test_consumption(market, sectors): """Test consumption output quantity.""" - result = consumption(market, mock_sectors, 2010) + result = consumption(market, sectors, YEAR) assert isinstance(result, pd.DataFrame) - assert not result.empty - # Market quantities include timeslice-related dimensions expected_cols = { "region", "commodity", @@ -43,12 +81,10 @@ def test_consumption(market, mock_sectors): assert set(result.columns) == expected_cols -def test_supply(market, mock_sectors): +def test_supply(market, sectors): """Test supply output quantity.""" - result = supply(market, mock_sectors, 2010) + result = supply(market, sectors, YEAR) assert isinstance(result, pd.DataFrame) - assert not result.empty - # Market quantities include timeslice-related dimensions expected_cols = { "region", "commodity", @@ -62,12 +98,10 @@ def test_supply(market, mock_sectors): assert set(result.columns) == expected_cols -def test_prices(market, mock_sectors): +def test_prices(market, sectors): """Test prices output quantity.""" - result = prices(market, mock_sectors, 2010) + result = prices(market, sectors, YEAR) assert isinstance(result, pd.DataFrame) - assert not result.empty - # Market quantities include timeslice-related dimensions expected_cols = { "region", "commodity", @@ -81,11 +115,10 @@ def test_prices(market, mock_sectors): assert set(result.columns) == expected_cols -def test_capacity(market, mock_sectors): +def test_capacity(market, sectors): """Test capacity output quantity.""" - result = capacity(market, mock_sectors, 2010) + result = capacity(market, sectors, YEAR) assert isinstance(result, pd.DataFrame) - assert not result.empty expected_cols = { "technology", "agent", @@ -93,16 +126,17 @@ def test_capacity(market, mock_sectors): "sector", "year", "capacity", + "region", + "asset", + "installed", } assert set(result.columns) == expected_cols -def test_fuel_costs(market, mock_sectors): +def test_fuel_costs(market, sectors): """Test fuel costs output quantity.""" - result = metric_fuel_costs(market, mock_sectors, 2010) + result = metric_fuel_costs(market, sectors, YEAR) assert isinstance(result, pd.DataFrame) - assert not result.empty - # Cost quantities include timeslice-related dimensions and region expected_cols = { "technology", "agent", @@ -114,37 +148,34 @@ def test_fuel_costs(market, mock_sectors): "day", "hour", "timeslice", + "asset", "fuel_consumption_costs", } assert set(result.columns) == expected_cols -def test_capital_costs(market, mock_sectors): +def test_capital_costs(market, sectors): """Test capital costs output quantity.""" - result = metric_capital_costs(market, mock_sectors, 2010) + result = metric_capital_costs(market, sectors, YEAR) assert isinstance(result, pd.DataFrame) - assert not result.empty - # Capital costs include technology attributes expected_cols = { "technology", "agent", "category", "sector", "year", - "tech_type", - "fuel", "capital_costs", "region", + "asset", + "installed", } assert set(result.columns) == expected_cols -def test_emission_costs(market, mock_sectors): +def test_emission_costs(market, sectors): """Test emission costs output quantity.""" - result = metric_emission_costs(market, mock_sectors, 2010) + result = metric_emission_costs(market, sectors, YEAR) assert isinstance(result, pd.DataFrame) - assert not result.empty - # Cost quantities include timeslice-related dimensions and region expected_cols = { "technology", "agent", @@ -157,16 +188,15 @@ def test_emission_costs(market, mock_sectors): "hour", "timeslice", "emission_costs", + "asset", } assert set(result.columns) == expected_cols -def test_lcoe(market, mock_sectors): +def test_lcoe(market, sectors): """Test LCOE output quantity.""" - result = metric_lcoe(market, mock_sectors, 2010) + result = metric_lcoe(market, sectors, YEAR) assert isinstance(result, pd.DataFrame) - assert not result.empty - # Cost quantities include timeslice-related dimensions and technology attributes expected_cols = { "technology", "agent", @@ -185,12 +215,10 @@ def test_lcoe(market, mock_sectors): assert set(result.columns) == expected_cols -def test_eac(market, mock_sectors): +def test_eac(market, sectors): """Test EAC output quantity.""" - result = metric_eac(market, mock_sectors, 2010) + result = metric_eac(market, sectors, YEAR) assert isinstance(result, pd.DataFrame) - assert not result.empty - # Cost quantities include timeslice-related dimensions and technology attributes expected_cols = { "technology", "agent",