donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

removed id creation on scrape

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2026-01-14 14:28:16 +01:00
parent 4ea0c78d3d
commit 93fbefc9d4
11 changed files with 107 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
integrity/checkpoint_dependencies.toml
View File

@@ -16,10 +16,14 @@ depends_on = ["lei_figi_mapping_complete"]
# CLEANSING STAGE (Depends on collection)
# ============================================================================
[checkpoints.yahoo_companies_cleansed]
description = "Company data cleansed and validated"
[checkpoints.yahoo_companies_cleansed_no_data]
description = "Companies cleansed of data with no Yahoo results"
depends_on = ["securities_data_complete"]
[checkpoints.yahoo_companies_cleansed_low_profile]
description = "Companies cleansed of low profile (insufficient market cap/price data)"
depends_on = ["yahoo_companies_cleansed_no_data"]
# ============================================================================
# ENRICHMENT GROUP (All depend on cleansed companies)
# ============================================================================
@@ -31,7 +35,7 @@ members = [
"yahoo_options_enrichment_complete",
"yahoo_chart_enrichment_complete"
]
depends_on = ["yahoo_companies_cleansed"]
depends_on = ["yahoo_companies_cleansed_low_profile"]
[checkpoints.yahoo_events_enrichment_complete]
description = "Corporate events enriched for all companies"