Initial commit: Energy test data generation pipeline
Add complete test data preparation system for energy trading strategy demo. Includes configuration, data generation scripts, and validation tools for 7 datasets covering electricity prices, battery capacity, renewable/conventional generation, load profiles, data centers, and mining data. Excluded from git: Actual parquet data files (data/raw/, data/processed/) can be regenerated using the provided scripts. Datasets: - electricity_prices: Day-ahead and real-time prices (5 regions) - battery_capacity: Storage system charge/discharge cycles - renewable_generation: Solar, wind, hydro with forecast errors - conventional_generation: Gas, coal, nuclear plant outputs - load_profiles: Regional demand with weather correlations - data_centers: Power demand profiles including mining operations - mining_data: Hashrate, price, profitability (mempool.space API)
This commit is contained in:
89
data/metadata/generation_metadata.json
Normal file
89
data/metadata/generation_metadata.json
Normal file
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"generated_at": "2026-02-10T16:10:43.522420",
|
||||
"datasets": {
|
||||
"battery_capacity": {
|
||||
"rows": 144010,
|
||||
"columns": [
|
||||
"timestamp",
|
||||
"battery_id",
|
||||
"capacity_mwh",
|
||||
"charge_level_mwh",
|
||||
"charge_rate_mw",
|
||||
"discharge_rate_mw",
|
||||
"efficiency"
|
||||
],
|
||||
"memory_usage_mb": 15.38205337524414,
|
||||
"dtypes": {
|
||||
"timestamp": "datetime64[ns]",
|
||||
"battery_id": "object",
|
||||
"capacity_mwh": "float64",
|
||||
"charge_level_mwh": "float64",
|
||||
"charge_rate_mw": "float64",
|
||||
"discharge_rate_mw": "float64",
|
||||
"efficiency": "float64"
|
||||
}
|
||||
},
|
||||
"renewable_generation": {
|
||||
"rows": 216015,
|
||||
"columns": [
|
||||
"timestamp",
|
||||
"source",
|
||||
"plant_id",
|
||||
"generation_mw",
|
||||
"forecast_mw",
|
||||
"actual_mw",
|
||||
"capacity_factor"
|
||||
],
|
||||
"memory_usage_mb": 34.472124099731445,
|
||||
"dtypes": {
|
||||
"timestamp": "datetime64[ns]",
|
||||
"source": "object",
|
||||
"plant_id": "object",
|
||||
"generation_mw": "float64",
|
||||
"forecast_mw": "float64",
|
||||
"actual_mw": "float64",
|
||||
"capacity_factor": "float64"
|
||||
}
|
||||
},
|
||||
"conventional_generation": {
|
||||
"rows": 144010,
|
||||
"columns": [
|
||||
"timestamp",
|
||||
"plant_id",
|
||||
"fuel_type",
|
||||
"generation_mw",
|
||||
"marginal_cost",
|
||||
"heat_rate"
|
||||
],
|
||||
"memory_usage_mb": 26.149402618408203,
|
||||
"dtypes": {
|
||||
"timestamp": "datetime64[ns]",
|
||||
"plant_id": "object",
|
||||
"fuel_type": "object",
|
||||
"generation_mw": "float64",
|
||||
"marginal_cost": "float64",
|
||||
"heat_rate": "float64"
|
||||
}
|
||||
},
|
||||
"data_centers": {
|
||||
"rows": 72005,
|
||||
"columns": [
|
||||
"timestamp",
|
||||
"data_center_id",
|
||||
"location",
|
||||
"power_demand_mw",
|
||||
"max_bid_price",
|
||||
"client_type"
|
||||
],
|
||||
"memory_usage_mb": 14.585489273071289,
|
||||
"dtypes": {
|
||||
"timestamp": "datetime64[ns]",
|
||||
"data_center_id": "object",
|
||||
"location": "object",
|
||||
"power_demand_mw": "float64",
|
||||
"max_bid_price": "float64",
|
||||
"client_type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user