Duckdb with sqlalchemy

pyproject.toml

@@ -35,7 +35,11 @@ dependencies = [
     "xlrd",
     "mypy-extensions",
     "pypubsub",
-    "tomlkit"
+    "tomlkit",
+    "duckdb",
+    "fsspec",
+    "sqlalchemy",
+    "duckdb-engine"
 ]
 dynamic = ["version"]
 

src/muse/data/example/default_new_input/agent_objectives.csv

@@ -0,0 +1,3 @@
+agent_id,objective,objective_data,objective_sort
+Agent1,LCOE,1,TRUE
+Agent2,LCOE,1,TRUE

src/muse/data/example/default_new_input/agent_pairs.csv

@@ -0,0 +1,2 @@
+id,new_agent_id,retrofit_agent_id,quantity
+A1,Agent1,Agent2,1

src/muse/data/example/default_new_input/agent_regions.csv

@@ -0,0 +1,2 @@
+agent_pair_id,region_id
+A1,R1

src/muse/data/example/default_new_input/agents.csv

@@ -0,0 +1,3 @@
+id,description,type,maturity_threshold,annual_cost_limit,search_rule,decision_rule
+Agent1,New agent for A1,new,-1,inf,all,single
+Agent2,Retrofit agent for A1,retrofit,-1,inf,all,single

src/muse/data/example/default_new_input/assets.csv

@@ -0,0 +1,8 @@
+agent_id,process_id,region_id,year,capacity
+Agent2,gassupply1,R1,2020,15
+Agent2,gassupply1,R1,2025,15
+Agent2,gassupply1,R1,2030,7.5
+Agent2,gasCCGT,R1,2020,1
+Agent2,gasCCGT,R1,2025,1
+Agent2,gasboiler,R1,2020,10
+Agent2,gasboiler,R1,2025,5

src/muse/data/example/default_new_input/commodities.csv

@@ -0,0 +1,6 @@
+name,description,type,unit
+electricity,Electricity,energy,PJ
+gas,Gas,energy,PJ
+heat,Heat,energy,PJ
+wind,Wind,energy,PJ
+CO2f,Carbon dioxide,energy,kt

src/muse/data/example/default_new_input/commodity_costs.csv

@@ -0,0 +1,58 @@
+commodity_id,region_id,year,value
+electricity,R1,2010,14.81481472
+electricity,R1,2015,17.89814806
+electricity,R1,2020,19.5
+electricity,R1,2025,21.93518528
+electricity,R1,2030,26.50925917
+electricity,R1,2035,26.51851861
+electricity,R1,2040,23.85185194
+electricity,R1,2045,23.97222222
+electricity,R1,2050,24.06481472
+electricity,R1,2055,25.3425925
+electricity,R1,2060,25.53703694
+electricity,R1,2065,25.32407417
+electricity,R1,2070,23.36111111
+electricity,R1,2075,22.27777778
+electricity,R1,2080,22.25925917
+electricity,R1,2085,22.17592583
+electricity,R1,2090,22.03703694
+electricity,R1,2095,21.94444444
+electricity,R1,2100,21.39814806
+gas,R1,2010,6.6759
+gas,R1,2015,6.914325
+gas,R1,2020,7.15275
+gas,R1,2025,8.10645
+gas,R1,2030,9.06015
+gas,R1,2035,9.2191
+gas,R1,2040,9.37805
+gas,R1,2045,9.193829337
+gas,R1,2050,9.009608674
+gas,R1,2055,8.832625604
+gas,R1,2060,8.655642534
+gas,R1,2065,8.485612708
+gas,R1,2070,8.315582883
+gas,R1,2075,8.152233126
+gas,R1,2080,7.988883368
+gas,R1,2085,7.831951236
+gas,R1,2090,7.675019103
+gas,R1,2095,7.524252461
+gas,R1,2100,7.373485819
+CO2f,R1,2010,0
+CO2f,R1,2015,0.052913851
+CO2f,R1,2020,0.08314119
+CO2f,R1,2025,0.120069795
+CO2f,R1,2030,0.156998399
+CO2f,R1,2035,0.214877567
+CO2f,R1,2040,0.272756734
+CO2f,R1,2045,0.35394801
+CO2f,R1,2050,0.435139285
+CO2f,R1,2055,0.542365578
+CO2f,R1,2060,0.649591871
+CO2f,R1,2065,0.780892624
+CO2f,R1,2070,0.912193378
+CO2f,R1,2075,1.078321687
+CO2f,R1,2080,1.244449995
+CO2f,R1,2085,1.4253503
+CO2f,R1,2090,1.606250604
+CO2f,R1,2095,1.73877515
+CO2f,R1,2100,1.871299697

src/muse/data/example/default_new_input/commodity_trade.csv

@@ -0,0 +1 @@
+commodity_id,region_id,year,import,export

src/muse/data/example/default_new_input/demand.csv

@@ -0,0 +1,3 @@
+commodity_id,region_id,year,demand
+heat,R1,2020,10
+heat,R1,2050,30

src/muse/data/example/default_new_input/demand_slicing.csv

@@ -0,0 +1,7 @@
+commodity_id,region_id,year,timeslice,fraction
+heat,R1,,night,0.1
+heat,R1,,morning,0.15
+heat,R1,,afternoon,0.1
+heat,R1,,early-peak,0.15
+heat,R1,,late-peak,0.3
+heat,R1,,evening,0.2

src/muse/data/example/default_new_input/process_availabilities.csv

@@ -0,0 +1,6 @@
+process_id,region_id,year,timeslice,lim_type,value
+gassupply1,,,ALL,UP,0.9
+gasCCGT,,,ALL,UP,0.9
+windturbine,,,ALL,UP,0.4
+gasboiler,,,ALL,UP,1
+heatpump,,,ALL,UP,1

src/muse/data/example/default_new_input/process_flows.csv

@@ -0,0 +1,12 @@
+process_id,commodity_id,region_id,year,flow
+gassupply1,gas,,,1
+gasCCGT,gas,,,-1.67
+gasCCGT,electricity,,,1
+gasCCGT,CO2f,,,91.67
+windturbine,wind,,,-1
+windturbine,electricity,,,1
+gasboiler,gas,,,-1.16
+gasboiler,heat,,,1
+gasboiler,CO2f,,,64.71
+heatpump,electricity,,,-0.4
+heatpump,heat,,,1

src/muse/data/example/default_new_input/process_parameters.csv

@@ -0,0 +1,6 @@
+process_id,region_id,year,cap_par,cap_exp,fix_par,fix_exp,var_par,var_exp,max_capacity_addition,max_capacity_growth,total_capacity_limit,life,scaling_size,efficiency,discount_rate
+gassupply1,,,0,1,0,1,2.55,1,5,1,60,35,0.00000189,86,0.1
+gasCCGT,,,23.78234399,1,0,1,0,1,2,1,60,35,0.00000189,86,0.1
+windturbine,,,36.30771182,1,0,1,0,1,2,1,60,25,0.00000189,86,0.1
+gasboiler,,,3.8,1,0,1,0,1,10,0.02,60,10,0.00000189,86,0.1
+heatpump,,,8.866667,1,0,1,0,1,10,0.02,60,10,0.00000189,86,0.1

src/muse/data/example/default_new_input/process_regions.csv

@@ -0,0 +1,6 @@
+process_id,region_id
+gassupply1,R1
+gasCCGT,R1
+windturbine,R1
+gasboiler,R1
+heatpump,R1

src/muse/data/example/default_new_input/processes.csv

@@ -0,0 +1,6 @@
+id,sector_id,type,fuel,end_use,level
+gassupply1,gas,energy,gas,gas,fixed
+gasCCGT,power,energy,gas,electricity,fixed
+windturbine,power,energy,wind,electricity,fixed
+gasboiler,residential,energy,gas,heat,fixed
+heatpump,residential,energy,electricity,heat,fixed

src/muse/data/example/default_new_input/regions.csv

@@ -0,0 +1,2 @@
+id,description
+R1,Region 1

src/muse/data/example/default_new_input/sectors.csv

@@ -0,0 +1,4 @@
+id,description
+gas,Gas sector
+power,Power sector
+residential,Residential sector

src/muse/data/example/default_new_input/time_slices.csv

@@ -0,0 +1,7 @@
+season,day,time_of_day,fraction
+all,all,night,0.1667
+all,all,morning,0.1667
+all,all,afternoon,0.1667
+all,all,early-peak,0.1667
+all,all,late-peak,0.1667
+all,all,evening,0.1667

src/muse/examples.py

@@ -52,6 +52,7 @@
     "multiple_agents",
     "minimum_service",
     "trade",
+    "default_new_input",
 ]
 
 
@@ -137,6 +138,8 @@ def copy_model(
         _copy_minimum_service(path)
     elif name.lower() == "trade":
         _copy_trade(path)
+    elif name.lower() == "default_new_input":
+        _copy_default_new_input(path)
     return path
 
 
@@ -316,6 +319,12 @@ def update_lpsolver(data):
     modify_toml(path / "settings.toml", update_lpsolver)
 
 
+def _copy_default_new_input(path: Path):
+    from shutil import copytree
+
+    copytree(example_data_dir() / "default_new_input", path)
+
+
 def _copy_default_timeslice(path: Path):
     copytree(example_data_dir() / "default_timeslice", path)
 

src/muse/new_input/readers.py

@@ -0,0 +1,89 @@
+import numpy as np
+import xarray as xr
+from sqlalchemy import CheckConstraint, ForeignKey
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+
+
+class TableBase(DeclarativeBase):
+    pass
+
+
+class Regions(TableBase):
+    __tablename__ = "regions"
+
+    name: Mapped[str] = mapped_column(primary_key=True)
+
+
+class Commodities(TableBase):
+    __tablename__ = "commodities"
+
+    name: Mapped[str] = mapped_column(primary_key=True)
+    type: Mapped[str] = mapped_column(
+        CheckConstraint("type IN ('energy', 'service', 'material', 'environmental')")
+    )
+    unit: Mapped[str]
+
+
+class Demand(TableBase):
+    __tablename__ = "demand"
+
+    year: Mapped[int] = mapped_column(primary_key=True, autoincrement=False)
+    commodity: Mapped[Commodities] = mapped_column(
+        ForeignKey("commodities.name"),
+        primary_key=True,
+        info=dict(header="commodity_name"),
+    )
+    region: Mapped[Regions] = mapped_column(
+        ForeignKey("regions.name"), primary_key=True
+    )
+    demand: Mapped[float]
+
+
+def read_inputs(data_dir):
+    from sqlalchemy import create_engine
+
+    engine = create_engine("duckdb:///:memory:")
+    TableBase.metadata.create_all(engine)
+    con = engine.raw_connection().driver_connection
+
+    with open(data_dir / "regions.csv") as f:
+        regions = read_csv(f, Regions, con)  # noqa: F841
+
+    with open(data_dir / "commodities.csv") as f:
+        commodities = read_csv(f, Commodities, con)
+
+    with open(data_dir / "demand.csv") as f:
+        demand = read_csv(f, Demand, con)  # noqa: F841
+
+    data = {}
+    data["global_commodities"] = calculate_global_commodities(commodities)
+    return data
+
+
+def read_csv(buffer_, table_class, con):
+    table_name = table_class.__tablename__
+    columns = ", ".join(
+        column.info.get("header", column.name)
+        for column in table_class.__table__.columns
+    )
+
+    rel = con.read_csv(buffer_, header=True, delimiter=",")  # noqa: F841
+    con.execute(f"INSERT INTO {table_name} SELECT {columns} FROM rel")
+    return con.execute(f"SELECT * from {table_name}").fetchnumpy()
+
+
+def calculate_global_commodities(commodities):
+    names = commodities["name"].astype(np.dtype("str"))
+    types = commodities["type"].astype(np.dtype("str"))
+    units = commodities["unit"].astype(np.dtype("str"))
+
+    type_array = xr.DataArray(
+        data=types, dims=["commodity"], coords=dict(commodity=names)
+    )
+
+    unit_array = xr.DataArray(
+        data=units, dims=["commodity"], coords=dict(commodity=names)
+    )
+
+    data = xr.Dataset(data_vars=dict(type=type_array, unit=unit_array))
+    return data

tests/test_fullsim_regression.py

@@ -5,6 +5,9 @@
 
 from muse.examples import AVAILABLE_EXAMPLES
 
+# temporary skip for default_new_input as this is not yet working
+AVAILABLE_EXAMPLES.remove("default_new_input")
+
 
 @mark.regression
 @mark.example