Add transmission datasets and update mining data
Add two new static datasets for cross-region arbitrage calculations: - transmission_capacity: region-to-region capacity limits (20 rows) - transmission_cost: transmission costs per path (20 rows) Update mining dataset with EUR pricing and power metrics: - Change btc_price_usd to btc_price_eur - Add power_efficiency_th_per_mw, power_demand_mw - Add revenue_eur_per_mwh, profit_eur_per_mwh - Remove mining_profitability column Changes include: - scripts/02_fetch_historical.py: rewrite fetch_bitcoin_mining_data() - scripts/01_generate_synthetic.py: add transmission data generators - config/data_config.yaml: add transmission config, update bitcoin config - config/schema.yaml: add 2 new schemas, update bitcoin_mining schema - scripts/03_process_merge.py: add 2 new datasets - scripts/04_validate.py: add 2 new datasets - test/test_data.py: update for new datasets and bitcoin price reference Total datasets: 9 (734,491 rows, 17.89 MB)
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"processed_at": "2026-02-10T16:10:49.295018+00:00",
|
||||
"total_datasets": 7,
|
||||
"total_size_mb": 16.977967262268066,
|
||||
"processed_at": "2026-02-10T17:49:27.237574+00:00",
|
||||
"total_datasets": 9,
|
||||
"total_size_mb": 17.2280216217041,
|
||||
"datasets": {
|
||||
"electricity_prices": {
|
||||
"path": "/home/user/energy-test-data/data/processed/electricity_prices.parquet",
|
||||
@@ -11,19 +11,19 @@
|
||||
},
|
||||
"battery_capacity": {
|
||||
"path": "/home/user/energy-test-data/data/processed/battery_capacity.parquet",
|
||||
"size_mb": 4.204527854919434,
|
||||
"size_mb": 4.204350471496582,
|
||||
"rows": 144010,
|
||||
"columns": 7
|
||||
},
|
||||
"renewable_generation": {
|
||||
"path": "/home/user/energy-test-data/data/processed/renewable_generation.parquet",
|
||||
"size_mb": 4.482715606689453,
|
||||
"size_mb": 4.483729362487793,
|
||||
"rows": 216015,
|
||||
"columns": 7
|
||||
},
|
||||
"conventional_generation": {
|
||||
"path": "/home/user/energy-test-data/data/processed/conventional_generation.parquet",
|
||||
"size_mb": 2.749570846557617,
|
||||
"size_mb": 2.7516822814941406,
|
||||
"rows": 144010,
|
||||
"columns": 6
|
||||
},
|
||||
@@ -35,14 +35,26 @@
|
||||
},
|
||||
"data_centers": {
|
||||
"path": "/home/user/energy-test-data/data/processed/data_centers.parquet",
|
||||
"size_mb": 1.0422554016113281,
|
||||
"size_mb": 1.0423173904418945,
|
||||
"rows": 72005,
|
||||
"columns": 6
|
||||
},
|
||||
"bitcoin_mining": {
|
||||
"path": "/home/user/energy-test-data/data/processed/bitcoin_mining.parquet",
|
||||
"size_mb": 0.3613767623901367,
|
||||
"size_mb": 0.5998897552490234,
|
||||
"rows": 14401,
|
||||
"columns": 9
|
||||
},
|
||||
"transmission_capacity": {
|
||||
"path": "/home/user/energy-test-data/data/processed/transmission_capacity.parquet",
|
||||
"size_mb": 0.0039043426513671875,
|
||||
"rows": 20,
|
||||
"columns": 5
|
||||
},
|
||||
"transmission_cost": {
|
||||
"path": "/home/user/energy-test-data/data/processed/transmission_cost.parquet",
|
||||
"size_mb": 0.004627227783203125,
|
||||
"rows": 20,
|
||||
"columns": 6
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user