Add transmission datasets and update mining data

Add two new static datasets for cross-region arbitrage calculations:
- transmission_capacity: region-to-region capacity limits (20 rows)
- transmission_cost: transmission costs per path (20 rows)

Update mining dataset with EUR pricing and power metrics:
- Change btc_price_usd to btc_price_eur
- Add power_efficiency_th_per_mw, power_demand_mw
- Add revenue_eur_per_mwh, profit_eur_per_mwh
- Remove mining_profitability column

Changes include:
- scripts/02_fetch_historical.py: rewrite fetch_bitcoin_mining_data()
- scripts/01_generate_synthetic.py: add transmission data generators
- config/data_config.yaml: add transmission config, update bitcoin config
- config/schema.yaml: add 2 new schemas, update bitcoin_mining schema
- scripts/03_process_merge.py: add 2 new datasets
- scripts/04_validate.py: add 2 new datasets
- test/test_data.py: update for new datasets and bitcoin price reference

Total datasets: 9 (734,491 rows, 17.89 MB)

data/metadata/final_metadata.json

@@ -1,7 +1,7 @@
 {
-  "processed_at": "2026-02-10T16:10:49.295018+00:00",
-  "total_datasets": 7,
-  "total_size_mb": 16.977967262268066,
+  "processed_at": "2026-02-10T17:49:27.237574+00:00",
+  "total_datasets": 9,
+  "total_size_mb": 17.2280216217041,
   "datasets": {
     "electricity_prices": {
       "path": "/home/user/energy-test-data/data/processed/electricity_prices.parquet",
@@ -11,19 +11,19 @@
     },
     "battery_capacity": {
       "path": "/home/user/energy-test-data/data/processed/battery_capacity.parquet",
-      "size_mb": 4.204527854919434,
+      "size_mb": 4.204350471496582,
       "rows": 144010,
       "columns": 7
     },
     "renewable_generation": {
       "path": "/home/user/energy-test-data/data/processed/renewable_generation.parquet",
-      "size_mb": 4.482715606689453,
+      "size_mb": 4.483729362487793,
       "rows": 216015,
       "columns": 7
     },
     "conventional_generation": {
       "path": "/home/user/energy-test-data/data/processed/conventional_generation.parquet",
-      "size_mb": 2.749570846557617,
+      "size_mb": 2.7516822814941406,
       "rows": 144010,
       "columns": 6
     },
@@ -35,14 +35,26 @@
     },
     "data_centers": {
       "path": "/home/user/energy-test-data/data/processed/data_centers.parquet",
-      "size_mb": 1.0422554016113281,
+      "size_mb": 1.0423173904418945,
       "rows": 72005,
       "columns": 6
     },
     "bitcoin_mining": {
       "path": "/home/user/energy-test-data/data/processed/bitcoin_mining.parquet",
-      "size_mb": 0.3613767623901367,
+      "size_mb": 0.5998897552490234,
       "rows": 14401,
+      "columns": 9
+    },
+    "transmission_capacity": {
+      "path": "/home/user/energy-test-data/data/processed/transmission_capacity.parquet",
+      "size_mb": 0.0039043426513671875,
+      "rows": 20,
+      "columns": 5
+    },
+    "transmission_cost": {
+      "path": "/home/user/energy-test-data/data/processed/transmission_cost.parquet",
+      "size_mb": 0.004627227783203125,
+      "rows": 20,
       "columns": 6
     }
   }

data/metadata/generation_metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-02-10T16:10:43.522420",
+  "generated_at": "2026-02-10T17:49:15.839052",
   "datasets": {
     "battery_capacity": {
       "rows": 144010,
@@ -84,6 +84,44 @@
         "max_bid_price": "float64",
         "client_type": "object"
       }
+    },
+    "transmission_capacity": {
+      "rows": 20,
+      "columns": [
+        "source_region",
+        "target_region",
+        "capacity_mw",
+        "direction",
+        "efficiency"
+      ],
+      "memory_usage_mb": 0.004016876220703125,
+      "dtypes": {
+        "source_region": "object",
+        "target_region": "object",
+        "capacity_mw": "float64",
+        "direction": "object",
+        "efficiency": "float64"
+      }
+    },
+    "transmission_cost": {
+      "rows": 20,
+      "columns": [
+        "source_region",
+        "target_region",
+        "cost_eur_mwh",
+        "loss_percent",
+        "congestion_surcharge_eur_mwh",
+        "fee_eur_mwh"
+      ],
+      "memory_usage_mb": 0.002986907958984375,
+      "dtypes": {
+        "source_region": "object",
+        "target_region": "object",
+        "cost_eur_mwh": "float64",
+        "loss_percent": "float64",
+        "congestion_surcharge_eur_mwh": "float64",
+        "fee_eur_mwh": "float64"
+      }
     }
   }
 }

data/metadata/validation_report.json

@@ -1,12 +1,12 @@
 {
-  "generated_at": "2026-02-10T16:10:53.614368",
+  "generated_at": "2026-02-10T17:49:31.592598",
   "summary": {
-    "total_datasets": 7,
-    "passed": 2,
+    "total_datasets": 9,
+    "passed": 4,
     "warnings": 5,
     "failed": 0,
-    "total_size_mb": 17.72,
-    "total_rows": 734451
+    "total_size_mb": 17.89,
+    "total_rows": 734491
   },
   "datasets": [
     {
@@ -64,13 +64,13 @@
         {
           "column": "efficiency",
           "rule": "min >= 0.5",
-          "violations": 36,
+          "violations": 56,
           "severity": "error"
         },
         {
           "column": "efficiency",
           "rule": "max <= 1.0",
-          "violations": 4371,
+          "violations": 4460,
           "severity": "error"
         }
       ],
@@ -111,7 +111,7 @@
         {
           "column": "capacity_factor",
           "rule": "max <= 1.0",
-          "violations": 6382,
+          "violations": 6284,
           "severity": "error"
         }
       ],
@@ -148,13 +148,13 @@
         {
           "column": "heat_rate",
           "rule": "min >= 5",
-          "violations": 29,
+          "violations": 27,
           "severity": "error"
         },
         {
           "column": "heat_rate",
           "rule": "max <= 15",
-          "violations": 867,
+          "violations": 845,
           "severity": "error"
         }
       ],
@@ -204,7 +204,7 @@
         {
           "column": "power_demand_mw",
           "rule": "min >= 0",
-          "violations": 137,
+          "violations": 135,
           "severity": "error"
         }
       ],
@@ -214,8 +214,8 @@
     {
       "dataset": "bitcoin_mining",
       "rows": 14401,
-      "columns": 6,
-      "memory_mb": 0.34,
+      "columns": 9,
+      "memory_mb": 0.51,
       "missing_values": {},
       "duplicated_rows": 0,
       "timestamp_continuity": {
@@ -226,14 +226,62 @@
       },
       "data_ranges": [
         {
-          "column": "btc_price_usd",
+          "column": "btc_price_eur",
           "rule": "min >= 1000",
-          "violations": 456,
+          "violations": 466,
+          "severity": "error"
+        },
+        {
+          "column": "power_demand_mw",
+          "rule": "min >= 10",
+          "violations": 14401,
+          "severity": "error"
+        },
+        {
+          "column": "revenue_eur_per_mwh",
+          "rule": "min >= 0",
+          "violations": 359,
+          "severity": "error"
+        },
+        {
+          "column": "revenue_eur_per_mwh",
+          "rule": "max <= 500",
+          "violations": 13959,
           "severity": "error"
         }
       ],
       "data_types": [],
       "status": "warning"
+    },
+    {
+      "dataset": "transmission_capacity",
+      "rows": 20,
+      "columns": 5,
+      "memory_mb": 0.0,
+      "missing_values": {},
+      "duplicated_rows": 0,
+      "timestamp_continuity": {
+        "status": "skipped",
+        "reason": "no timestamp column"
+      },
+      "data_ranges": [],
+      "data_types": [],
+      "status": "pass"
+    },
+    {
+      "dataset": "transmission_cost",
+      "rows": 20,
+      "columns": 6,
+      "memory_mb": 0.0,
+      "missing_values": {},
+      "duplicated_rows": 0,
+      "timestamp_continuity": {
+        "status": "skipped",
+        "reason": "no timestamp column"
+      },
+      "data_ranges": [],
+      "data_types": [],
+      "status": "pass"
     }
   ]
 }