Compare commits
6 Commits
7b680f960f
...
2fe06a9d88
| Author | SHA1 | Date | |
|---|---|---|---|
| 2fe06a9d88 | |||
| 9db4320c40 | |||
| eeae94e041 | |||
| e57a013224 | |||
| bbc19f2110 | |||
| 9cfcae84ea |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -17,6 +17,9 @@ target/
|
|||||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
#.idea/
|
#.idea/
|
||||||
|
|
||||||
|
# env
|
||||||
|
.env
|
||||||
|
|
||||||
|
|
||||||
# Added by cargo
|
# Added by cargo
|
||||||
|
|
||||||
@@ -30,3 +33,4 @@ target/
|
|||||||
/corporate_events*
|
/corporate_events*
|
||||||
/corporate_prices*
|
/corporate_prices*
|
||||||
/corporate_event_changes*
|
/corporate_event_changes*
|
||||||
|
/data*
|
||||||
281
Cargo.lock
generated
281
Cargo.lock
generated
@@ -8,6 +8,17 @@ version = "2.0.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "aes"
|
||||||
|
version = "0.8.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"cipher",
|
||||||
|
"cpufeatures",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ahash"
|
name = "ahash"
|
||||||
version = "0.7.8"
|
version = "0.7.8"
|
||||||
@@ -71,6 +82,15 @@ version = "1.0.100"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
|
checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "arbitrary"
|
||||||
|
version = "1.4.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
|
||||||
|
dependencies = [
|
||||||
|
"derive_arbitrary",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "arrayvec"
|
name = "arrayvec"
|
||||||
version = "0.7.6"
|
version = "0.7.6"
|
||||||
@@ -228,6 +248,15 @@ version = "1.11.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3"
|
checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bzip2"
|
||||||
|
version = "0.6.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c"
|
||||||
|
dependencies = [
|
||||||
|
"libbz2-rs-sys",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cc"
|
name = "cc"
|
||||||
version = "1.2.46"
|
version = "1.2.46"
|
||||||
@@ -235,6 +264,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "b97463e1064cb1b1c1384ad0a0b9c8abd0988e2a91f52606c80ef14aadb63e36"
|
checksum = "b97463e1064cb1b1c1384ad0a0b9c8abd0988e2a91f52606c80ef14aadb63e36"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"find-msvc-tools",
|
"find-msvc-tools",
|
||||||
|
"jobserver",
|
||||||
|
"libc",
|
||||||
"shlex",
|
"shlex",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -275,6 +306,16 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cipher"
|
||||||
|
version = "0.4.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||||
|
dependencies = [
|
||||||
|
"crypto-common",
|
||||||
|
"inout",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "compression-codecs"
|
name = "compression-codecs"
|
||||||
version = "0.4.32"
|
version = "0.4.32"
|
||||||
@@ -293,6 +334,12 @@ version = "0.4.30"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3a9b614a5787ef0c8802a55766480563cb3a93b435898c422ed2a359cf811582"
|
checksum = "3a9b614a5787ef0c8802a55766480563cb3a93b435898c422ed2a359cf811582"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "constant_time_eq"
|
||||||
|
version = "0.3.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cookie"
|
name = "cookie"
|
||||||
version = "0.16.2"
|
version = "0.16.2"
|
||||||
@@ -378,6 +425,21 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crc"
|
||||||
|
version = "3.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675"
|
||||||
|
dependencies = [
|
||||||
|
"crc-catalog",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crc-catalog"
|
||||||
|
version = "2.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crc32fast"
|
name = "crc32fast"
|
||||||
version = "1.5.0"
|
version = "1.5.0"
|
||||||
@@ -445,12 +507,39 @@ dependencies = [
|
|||||||
"syn 2.0.110",
|
"syn 2.0.110",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "csv"
|
||||||
|
version = "1.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"
|
||||||
|
dependencies = [
|
||||||
|
"csv-core",
|
||||||
|
"itoa",
|
||||||
|
"ryu",
|
||||||
|
"serde_core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "csv-core"
|
||||||
|
version = "0.1.13"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "data-encoding"
|
name = "data-encoding"
|
||||||
version = "2.9.0"
|
version = "2.9.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476"
|
checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "deflate64"
|
||||||
|
version = "0.1.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "deranged"
|
name = "deranged"
|
||||||
version = "0.5.5"
|
version = "0.5.5"
|
||||||
@@ -460,6 +549,17 @@ dependencies = [
|
|||||||
"powerfmt",
|
"powerfmt",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "derive_arbitrary"
|
||||||
|
version = "1.4.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.110",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "derive_more"
|
name = "derive_more"
|
||||||
version = "0.99.20"
|
version = "0.99.20"
|
||||||
@@ -479,6 +579,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"block-buffer",
|
"block-buffer",
|
||||||
"crypto-common",
|
"crypto-common",
|
||||||
|
"subtle",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -501,6 +602,12 @@ dependencies = [
|
|||||||
"litrs",
|
"litrs",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dotenvy"
|
||||||
|
version = "0.15.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "dtoa"
|
name = "dtoa"
|
||||||
version = "1.0.10"
|
version = "1.0.10"
|
||||||
@@ -559,8 +666,12 @@ version = "0.1.0"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"chrono",
|
"chrono",
|
||||||
|
"csv",
|
||||||
|
"dotenvy",
|
||||||
"fantoccini",
|
"fantoccini",
|
||||||
|
"flate2",
|
||||||
"futures",
|
"futures",
|
||||||
|
"rand 0.9.2",
|
||||||
"rayon",
|
"rayon",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"scraper",
|
"scraper",
|
||||||
@@ -570,6 +681,7 @@ dependencies = [
|
|||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
"yfinance-rs",
|
"yfinance-rs",
|
||||||
|
"zip",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -621,6 +733,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb"
|
checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"crc32fast",
|
"crc32fast",
|
||||||
|
"libz-rs-sys",
|
||||||
"miniz_oxide",
|
"miniz_oxide",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -854,6 +967,15 @@ version = "0.5.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hmac"
|
||||||
|
version = "0.12.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
|
||||||
|
dependencies = [
|
||||||
|
"digest",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "html5ever"
|
name = "html5ever"
|
||||||
version = "0.27.0"
|
version = "0.27.0"
|
||||||
@@ -1205,6 +1327,15 @@ dependencies = [
|
|||||||
"hashbrown 0.16.1",
|
"hashbrown 0.16.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "inout"
|
||||||
|
version = "0.1.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||||
|
dependencies = [
|
||||||
|
"generic-array",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ipnet"
|
name = "ipnet"
|
||||||
version = "2.11.0"
|
version = "2.11.0"
|
||||||
@@ -1262,6 +1393,16 @@ version = "1.0.15"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
|
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "jobserver"
|
||||||
|
version = "0.1.34"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
|
||||||
|
dependencies = [
|
||||||
|
"getrandom 0.3.4",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "js-sys"
|
name = "js-sys"
|
||||||
version = "0.3.82"
|
version = "0.3.82"
|
||||||
@@ -1278,12 +1419,27 @@ version = "1.5.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libbz2-rs-sys"
|
||||||
|
version = "0.2.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libc"
|
name = "libc"
|
||||||
version = "0.2.177"
|
version = "0.2.177"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
|
checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libz-rs-sys"
|
||||||
|
version = "0.5.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd"
|
||||||
|
dependencies = [
|
||||||
|
"zlib-rs",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "linux-raw-sys"
|
name = "linux-raw-sys"
|
||||||
version = "0.11.0"
|
version = "0.11.0"
|
||||||
@@ -1323,6 +1479,16 @@ version = "0.1.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
|
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lzma-rust2"
|
||||||
|
version = "0.13.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c60a23ffb90d527e23192f1246b14746e2f7f071cb84476dd879071696c18a4a"
|
||||||
|
dependencies = [
|
||||||
|
"crc",
|
||||||
|
"sha2",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mac"
|
name = "mac"
|
||||||
version = "0.1.1"
|
version = "0.1.1"
|
||||||
@@ -1624,6 +1790,16 @@ dependencies = [
|
|||||||
"windows-link",
|
"windows-link",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pbkdf2"
|
||||||
|
version = "0.12.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
|
||||||
|
dependencies = [
|
||||||
|
"digest",
|
||||||
|
"hmac",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "percent-encoding"
|
name = "percent-encoding"
|
||||||
version = "2.3.2"
|
version = "2.3.2"
|
||||||
@@ -1781,6 +1957,12 @@ version = "0.2.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
|
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ppmd-rust"
|
||||||
|
version = "1.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d558c559f0450f16f2a27a1f017ef38468c1090c9ce63c8e51366232d53717b4"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ppv-lite86"
|
name = "ppv-lite86"
|
||||||
version = "0.2.21"
|
version = "0.2.21"
|
||||||
@@ -2126,6 +2308,7 @@ dependencies = [
|
|||||||
"cookie 0.18.1",
|
"cookie 0.18.1",
|
||||||
"cookie_store",
|
"cookie_store",
|
||||||
"encoding_rs",
|
"encoding_rs",
|
||||||
|
"futures-channel",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"h2",
|
"h2",
|
||||||
@@ -2520,6 +2703,17 @@ dependencies = [
|
|||||||
"digest",
|
"digest",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sha2"
|
||||||
|
version = "0.10.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"cpufeatures",
|
||||||
|
"digest",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sharded-slab"
|
name = "sharded-slab"
|
||||||
version = "0.1.7"
|
version = "0.1.7"
|
||||||
@@ -3645,6 +3839,20 @@ name = "zeroize"
|
|||||||
version = "1.8.2"
|
version = "1.8.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
|
checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
|
||||||
|
dependencies = [
|
||||||
|
"zeroize_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zeroize_derive"
|
||||||
|
version = "1.4.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.110",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zerotrie"
|
name = "zerotrie"
|
||||||
@@ -3678,3 +3886,76 @@ dependencies = [
|
|||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.110",
|
"syn 2.0.110",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zip"
|
||||||
|
version = "6.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b"
|
||||||
|
dependencies = [
|
||||||
|
"aes",
|
||||||
|
"arbitrary",
|
||||||
|
"bzip2",
|
||||||
|
"constant_time_eq",
|
||||||
|
"crc32fast",
|
||||||
|
"deflate64",
|
||||||
|
"flate2",
|
||||||
|
"getrandom 0.3.4",
|
||||||
|
"hmac",
|
||||||
|
"indexmap",
|
||||||
|
"lzma-rust2",
|
||||||
|
"memchr",
|
||||||
|
"pbkdf2",
|
||||||
|
"ppmd-rust",
|
||||||
|
"sha1",
|
||||||
|
"time",
|
||||||
|
"zeroize",
|
||||||
|
"zopfli",
|
||||||
|
"zstd",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zlib-rs"
|
||||||
|
version = "0.5.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zopfli"
|
||||||
|
version = "0.8.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
|
||||||
|
dependencies = [
|
||||||
|
"bumpalo",
|
||||||
|
"crc32fast",
|
||||||
|
"log",
|
||||||
|
"simd-adler32",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zstd"
|
||||||
|
version = "0.13.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
|
||||||
|
dependencies = [
|
||||||
|
"zstd-safe",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zstd-safe"
|
||||||
|
version = "7.2.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
|
||||||
|
dependencies = [
|
||||||
|
"zstd-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zstd-sys"
|
||||||
|
version = "2.0.16+zstd.1.5.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
"pkg-config",
|
||||||
|
]
|
||||||
|
|||||||
11
Cargo.toml
11
Cargo.toml
@@ -17,7 +17,7 @@ categories = ["finance", "data-structures", "asynchronous"]
|
|||||||
tokio = { version = "1.38", features = ["full"] }
|
tokio = { version = "1.38", features = ["full"] }
|
||||||
|
|
||||||
# Web scraping & HTTP
|
# Web scraping & HTTP
|
||||||
reqwest = { version = "0.12", features = ["json", "gzip", "brotli", "deflate"] }
|
reqwest = { version = "0.12", features = ["json", "gzip", "brotli", "deflate", "blocking"] }
|
||||||
scraper = "0.19" # HTML parsing for Yahoo earnings pages
|
scraper = "0.19" # HTML parsing for Yahoo earnings pages
|
||||||
fantoccini = { version = "0.20", features = ["rustls-tls"] } # Headless Chrome for finanzen.net
|
fantoccini = { version = "0.20", features = ["rustls-tls"] } # Headless Chrome for finanzen.net
|
||||||
yfinance-rs = "0.7.2"
|
yfinance-rs = "0.7.2"
|
||||||
@@ -25,6 +25,15 @@ yfinance-rs = "0.7.2"
|
|||||||
# Serialization
|
# Serialization
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
|
csv = "1.3"
|
||||||
|
zip = "6.0.0"
|
||||||
|
flate2 = "1.1.5"
|
||||||
|
|
||||||
|
# Generating
|
||||||
|
rand = "0.9.2"
|
||||||
|
|
||||||
|
# Environment handling
|
||||||
|
dotenvy = "0.15"
|
||||||
|
|
||||||
# Date & time
|
# Date & time
|
||||||
chrono = { version = "0.4", features = ["serde"] }
|
chrono = { version = "0.4", features = ["serde"] }
|
||||||
|
|||||||
46
fx_rates.json
Normal file
46
fx_rates.json
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
{
|
||||||
|
"CHF": [
|
||||||
|
0.808996035919424,
|
||||||
|
"2025-11-25"
|
||||||
|
],
|
||||||
|
"JPY": [
|
||||||
|
0.0064,
|
||||||
|
"2025-11-25"
|
||||||
|
],
|
||||||
|
"INR": [
|
||||||
|
89.28571428571429,
|
||||||
|
"2025-11-25"
|
||||||
|
],
|
||||||
|
"GBp": [
|
||||||
|
0.7603406326034063,
|
||||||
|
"2025-11-25"
|
||||||
|
],
|
||||||
|
"AUD": [
|
||||||
|
1.5463120457708364,
|
||||||
|
"2025-11-25"
|
||||||
|
],
|
||||||
|
"SAR": [
|
||||||
|
3.750937734433609,
|
||||||
|
"2025-11-25"
|
||||||
|
],
|
||||||
|
"TWD": [
|
||||||
|
31.446540880503143,
|
||||||
|
"2025-11-25"
|
||||||
|
],
|
||||||
|
"CNY": [
|
||||||
|
7.087172218284904,
|
||||||
|
"2025-11-25"
|
||||||
|
],
|
||||||
|
"HKD": [
|
||||||
|
7.776049766718508,
|
||||||
|
"2025-11-25"
|
||||||
|
],
|
||||||
|
"CAD": [
|
||||||
|
1.4110342881332016,
|
||||||
|
"2025-11-25"
|
||||||
|
],
|
||||||
|
"EUR": [
|
||||||
|
0.8649022660439372,
|
||||||
|
"2025-11-25"
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -28,24 +28,3 @@ impl Config {
|
|||||||
future.format("%Y-%m-%d").to_string()
|
future.format("%Y-%m-%d").to_string()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_tickers() -> Vec<String> {
|
|
||||||
vec![
|
|
||||||
"JPM".to_string(), // XNYS
|
|
||||||
"MSFT".to_string(), // XNAS
|
|
||||||
"601398.SS".to_string(),// XSHG
|
|
||||||
"7203.T".to_string(), // XJPX
|
|
||||||
"0700.HK".to_string(), // XHKG
|
|
||||||
"ASML.AS".to_string(), // XAMS
|
|
||||||
"RELIANCE.BO".to_string(), // XBSE
|
|
||||||
"RELIANCE.NS".to_string(), // XNSE
|
|
||||||
"000001.SZ".to_string(),// XSHE
|
|
||||||
"SHOP.TO".to_string(), // XTSE
|
|
||||||
"AZN.L".to_string(), // XLON
|
|
||||||
"2330.TW".to_string(), // XTAI
|
|
||||||
"2222.SR".to_string(), // XSAU (note: uses .SR suffix)
|
|
||||||
"SAP.DE".to_string(), // XFRA
|
|
||||||
"NESN.SW".to_string(), // XSWX
|
|
||||||
"CSL.AX".to_string(), // XASX
|
|
||||||
]
|
|
||||||
}
|
|
||||||
194
src/corporate/aggregation.rs
Normal file
194
src/corporate/aggregation.rs
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
// src/corporate/aggregation.rs
|
||||||
|
use super::types::CompanyPrice;
|
||||||
|
use super::storage::*;
|
||||||
|
use tokio::fs;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct DayData {
|
||||||
|
sources: Vec<(CompanyPrice, String)>, // (price, source_ticker)
|
||||||
|
total_volume: u64,
|
||||||
|
vwap: f64,
|
||||||
|
open: f64,
|
||||||
|
high: f64,
|
||||||
|
low: f64,
|
||||||
|
close: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Aggregate price data from multiple exchanges, converting all to USD
|
||||||
|
pub async fn aggregate_best_price_data(lei: &str) -> anyhow::Result<()> {
|
||||||
|
let company_dir = get_company_dir(lei);
|
||||||
|
|
||||||
|
for timeframe in ["daily", "5min"].iter() {
|
||||||
|
let source_dir = company_dir.join(timeframe);
|
||||||
|
if !source_dir.exists() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut all_prices: Vec<(CompanyPrice, String)> = Vec::new();
|
||||||
|
let mut by_date_time: HashMap<String, DayData> = HashMap::new();
|
||||||
|
|
||||||
|
// Load all sources with their ticker names
|
||||||
|
let mut entries = tokio::fs::read_dir(&source_dir).await?;
|
||||||
|
let mut source_count = 0;
|
||||||
|
let mut sources_used = std::collections::HashSet::new();
|
||||||
|
|
||||||
|
while let Some(entry) = entries.next_entry().await? {
|
||||||
|
let source_dir_path = entry.path();
|
||||||
|
if !source_dir_path.is_dir() { continue; }
|
||||||
|
|
||||||
|
let source_ticker = source_dir_path
|
||||||
|
.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.unwrap_or("unknown")
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
let prices_path = source_dir_path.join("prices.json");
|
||||||
|
if !prices_path.exists() { continue; }
|
||||||
|
|
||||||
|
let content = tokio::fs::read_to_string(&prices_path).await?;
|
||||||
|
let mut prices: Vec<CompanyPrice> = serde_json::from_str(&content)?;
|
||||||
|
|
||||||
|
if !prices.is_empty() {
|
||||||
|
sources_used.insert(source_ticker.clone());
|
||||||
|
source_count += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
for price in prices {
|
||||||
|
all_prices.push((price, source_ticker.clone()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if all_prices.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
println!(" Aggregating from {} exchanges: {}",
|
||||||
|
sources_used.len(),
|
||||||
|
sources_used.iter()
|
||||||
|
.map(|s| s.as_str())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
);
|
||||||
|
|
||||||
|
// Group by date + time (for 5min) or just date
|
||||||
|
for (p, source) in all_prices {
|
||||||
|
let key = if timeframe == &"5min" && !p.time.is_empty() {
|
||||||
|
format!("{}_{}", p.date, p.time)
|
||||||
|
} else {
|
||||||
|
p.date.clone()
|
||||||
|
};
|
||||||
|
|
||||||
|
// Convert to USD immediately
|
||||||
|
let usd_rate = super::fx::get_usd_rate(&p.currency).await.unwrap_or(1.0);
|
||||||
|
|
||||||
|
let mut p_usd = p.clone();
|
||||||
|
p_usd.open *= usd_rate;
|
||||||
|
p_usd.high *= usd_rate;
|
||||||
|
p_usd.low *= usd_rate;
|
||||||
|
p_usd.close *= usd_rate;
|
||||||
|
p_usd.adj_close *= usd_rate;
|
||||||
|
p_usd.currency = "USD".to_string();
|
||||||
|
|
||||||
|
let entry = by_date_time.entry(key.clone()).or_insert(DayData {
|
||||||
|
sources: vec![],
|
||||||
|
total_volume: 0,
|
||||||
|
vwap: 0.0,
|
||||||
|
open: p_usd.open,
|
||||||
|
high: p_usd.high,
|
||||||
|
low: p_usd.low,
|
||||||
|
close: p_usd.close,
|
||||||
|
});
|
||||||
|
|
||||||
|
let volume = p.volume.max(1); // avoid div0
|
||||||
|
let vwap_contrib = p_usd.close * volume as f64;
|
||||||
|
|
||||||
|
entry.sources.push((p_usd.clone(), source));
|
||||||
|
entry.total_volume += volume;
|
||||||
|
entry.vwap += vwap_contrib;
|
||||||
|
|
||||||
|
// Use first open, last close, max high, min low
|
||||||
|
if entry.sources.len() == 1 {
|
||||||
|
entry.open = p_usd.open;
|
||||||
|
}
|
||||||
|
entry.close = p_usd.close;
|
||||||
|
entry.high = entry.high.max(p_usd.high);
|
||||||
|
entry.low = entry.low.min(p_usd.low);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finalize aggregated data
|
||||||
|
let mut aggregated: Vec<CompanyPrice> = Vec::new();
|
||||||
|
|
||||||
|
for (key, data) in by_date_time {
|
||||||
|
let vwap = data.vwap / data.total_volume as f64;
|
||||||
|
|
||||||
|
let (date, time) = if key.contains('_') {
|
||||||
|
let parts: Vec<&str> = key.split('_').collect();
|
||||||
|
(parts[0].to_string(), parts[1].to_string())
|
||||||
|
} else {
|
||||||
|
(key, "".to_string())
|
||||||
|
};
|
||||||
|
|
||||||
|
// Track which exchange contributed most volume
|
||||||
|
let best_source = data.sources.iter()
|
||||||
|
.max_by_key(|(p, _)| p.volume)
|
||||||
|
.map(|(_, src)| src.clone())
|
||||||
|
.unwrap_or_else(|| "unknown".to_string());
|
||||||
|
|
||||||
|
aggregated.push(CompanyPrice {
|
||||||
|
ticker: format!("{lei}@agg"), // Mark as aggregated
|
||||||
|
date,
|
||||||
|
time,
|
||||||
|
open: data.open,
|
||||||
|
high: data.high,
|
||||||
|
low: data.low,
|
||||||
|
close: data.close,
|
||||||
|
adj_close: vwap,
|
||||||
|
volume: data.total_volume,
|
||||||
|
currency: "USD".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
aggregated.sort_by_key(|p| (p.date.clone(), p.time.clone()));
|
||||||
|
|
||||||
|
// Save aggregated result
|
||||||
|
let agg_dir = company_dir.join("aggregated").join(timeframe);
|
||||||
|
fs::create_dir_all(&agg_dir).await?;
|
||||||
|
let path = agg_dir.join("prices.json");
|
||||||
|
fs::write(&path, serde_json::to_string_pretty(&aggregated)?).await?;
|
||||||
|
|
||||||
|
// Save aggregation metadata
|
||||||
|
let meta = AggregationMetadata {
|
||||||
|
lei: lei.to_string(), // ← CHANGE THIS
|
||||||
|
timeframe: timeframe.to_string(),
|
||||||
|
sources: sources_used.into_iter().collect(),
|
||||||
|
total_bars: aggregated.len(),
|
||||||
|
date_range: (
|
||||||
|
aggregated.first().map(|p| p.date.clone()).unwrap_or_default(),
|
||||||
|
aggregated.last().map(|p| p.date.clone()).unwrap_or_default(),
|
||||||
|
),
|
||||||
|
aggregated_at: chrono::Local::now().format("%Y-%m-%d %H:%M:%S").to_string(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let meta_path = agg_dir.join("metadata.json");
|
||||||
|
fs::write(&meta_path, serde_json::to_string_pretty(&meta)?).await?;
|
||||||
|
|
||||||
|
println!(" ✓ {} {} bars from {} sources (USD)",
|
||||||
|
aggregated.len(),
|
||||||
|
timeframe,
|
||||||
|
source_count
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Serialize, serde::Deserialize)]
|
||||||
|
struct AggregationMetadata {
|
||||||
|
lei: String,
|
||||||
|
timeframe: String,
|
||||||
|
sources: Vec<String>,
|
||||||
|
total_bars: usize,
|
||||||
|
date_range: (String, String),
|
||||||
|
aggregated_at: String,
|
||||||
|
}
|
||||||
51
src/corporate/fx.rs
Normal file
51
src/corporate/fx.rs
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
// src/corporate/fx.rs
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use reqwest;
|
||||||
|
use serde_json::Value;
|
||||||
|
use tokio::fs;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
static FX_CACHE_PATH: &str = "fx_rates.json";
|
||||||
|
|
||||||
|
pub async fn get_usd_rate(currency: &str) -> anyhow::Result<f64> {
|
||||||
|
if currency == "USD" {
|
||||||
|
return Ok(1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut cache: HashMap<String, (f64, String)> = if Path::new(FX_CACHE_PATH).exists() {
|
||||||
|
let content = fs::read_to_string(FX_CACHE_PATH).await?;
|
||||||
|
serde_json::from_str(&content).unwrap_or_default()
|
||||||
|
} else {
|
||||||
|
HashMap::new()
|
||||||
|
};
|
||||||
|
|
||||||
|
let today = chrono::Local::now().format("%Y-%m-%d").to_string();
|
||||||
|
if let Some((rate, date)) = cache.get(currency) {
|
||||||
|
if date == &today {
|
||||||
|
return Ok(*rate);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let symbol = format!("{}USD=X", currency);
|
||||||
|
let url = format!("https://query1.finance.yahoo.com/v8/finance/chart/{}?range=1d&interval=1d", symbol);
|
||||||
|
|
||||||
|
let json: Value = reqwest::Client::new()
|
||||||
|
.get(&url)
|
||||||
|
.header("User-Agent", "Mozilla/5.0")
|
||||||
|
.send()
|
||||||
|
.await?
|
||||||
|
.json()
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let close = json["chart"]["result"][0]["meta"]["regularMarketPrice"]
|
||||||
|
.as_f64()
|
||||||
|
.or_else(|| json["chart"]["result"][0]["indicators"]["quote"][0]["close"][0].as_f64())
|
||||||
|
.unwrap_or(1.0);
|
||||||
|
|
||||||
|
let rate = if currency == "JPY" || currency == "KRW" { close } else { 1.0 / close }; // inverse pairs
|
||||||
|
|
||||||
|
cache.insert(currency.to_string(), (rate, today.clone()));
|
||||||
|
let _ = fs::write(FX_CACHE_PATH, serde_json::to_string_pretty(&cache)?).await;
|
||||||
|
|
||||||
|
Ok(rate)
|
||||||
|
}
|
||||||
@@ -50,3 +50,21 @@ pub fn detect_changes(old: &CompanyEvent, new: &CompanyEvent, today: &str) -> Ve
|
|||||||
|
|
||||||
changes
|
changes
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn price_key(p: &CompanyPrice) -> String {
|
||||||
|
if p.time.is_empty() {
|
||||||
|
format!("{}|{}", p.ticker, p.date)
|
||||||
|
} else {
|
||||||
|
format!("{}|{}|{}", p.ticker, p.date, p.time)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn parse_float(s: &str) -> Option<f64> {
|
||||||
|
s.replace("--", "").replace(",", "").parse::<f64>().ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn parse_yahoo_date(s: &str) -> anyhow::Result<NaiveDate> {
|
||||||
|
NaiveDate::parse_from_str(s, "%B %d, %Y")
|
||||||
|
.or_else(|_| NaiveDate::parse_from_str(s, "%b %d, %Y"))
|
||||||
|
.map_err(|_| anyhow::anyhow!("Bad date: {s}"))
|
||||||
|
}
|
||||||
@@ -4,6 +4,9 @@ pub mod scraper;
|
|||||||
pub mod storage;
|
pub mod storage;
|
||||||
pub mod update;
|
pub mod update;
|
||||||
pub mod helpers;
|
pub mod helpers;
|
||||||
|
pub mod aggregation;
|
||||||
|
pub mod fx;
|
||||||
|
pub mod openfigi;
|
||||||
|
|
||||||
pub use types::*;
|
pub use types::*;
|
||||||
pub use update::run_full_update;
|
pub use update::run_full_update;
|
||||||
172
src/corporate/openfigi.rs
Normal file
172
src/corporate/openfigi.rs
Normal file
@@ -0,0 +1,172 @@
|
|||||||
|
// src/corporate/openfigi.rs
|
||||||
|
use super::{types::*};
|
||||||
|
use reqwest::Client as HttpClient;
|
||||||
|
use reqwest::header::{HeaderMap, HeaderValue};
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use std::collections::{HashMap, HashSet};
|
||||||
|
use tokio::time::{sleep, Duration};
|
||||||
|
use anyhow::Context;
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct OpenFigiClient {
|
||||||
|
client: HttpClient,
|
||||||
|
api_key: Option<String>,
|
||||||
|
has_key: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OpenFigiClient {
|
||||||
|
pub fn new() -> anyhow::Result<Self> {
|
||||||
|
let api_key = dotenvy::var("OPENFIGI_API_KEY").ok();
|
||||||
|
let has_key = api_key.is_some();
|
||||||
|
|
||||||
|
let mut builder = HttpClient::builder()
|
||||||
|
.user_agent("Mozilla/5.0 (compatible; OpenFIGI-Rust/1.0)")
|
||||||
|
.timeout(Duration::from_secs(30));
|
||||||
|
|
||||||
|
if let Some(key) = &api_key {
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert("X-OPENFIGI-APIKEY", HeaderValue::from_str(key)?);
|
||||||
|
builder = builder.default_headers(headers);
|
||||||
|
}
|
||||||
|
|
||||||
|
let client = builder.build().context("Failed to build HTTP client")?;
|
||||||
|
|
||||||
|
println!(
|
||||||
|
"OpenFIGI client initialized: {}",
|
||||||
|
if has_key { "with API key" } else { "no key (limited mode)" }
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(Self { client, api_key, has_key })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Batch-map ISINs to FIGI, filtering equities only
|
||||||
|
pub async fn map_isins_to_figi(&self, isins: &[String]) -> anyhow::Result<Vec<String>> {
|
||||||
|
if isins.is_empty() { return Ok(vec![]); }
|
||||||
|
|
||||||
|
let mut all_figis = Vec::new();
|
||||||
|
let chunk_size = if self.has_key { 100 } else { 5 };
|
||||||
|
|
||||||
|
for chunk in isins.chunks(chunk_size) {
|
||||||
|
let jobs: Vec<Value> = chunk.iter()
|
||||||
|
.map(|isin| json!({
|
||||||
|
"idType": "ID_ISIN",
|
||||||
|
"idValue": isin,
|
||||||
|
"marketSecDes": "Equity", // Pre-filter to equities
|
||||||
|
}))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let resp = self.client
|
||||||
|
.post("https://api.openfigi.com/v3/mapping")
|
||||||
|
.header("Content-Type", "application/json")
|
||||||
|
.json(&jobs)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let status = resp.status();
|
||||||
|
let headers = resp.headers().clone();
|
||||||
|
let body = resp.text().await.unwrap_or_default();
|
||||||
|
|
||||||
|
if status.is_client_error() || status.is_server_error() {
|
||||||
|
if status == 401 {
|
||||||
|
return Err(anyhow::anyhow!("Invalid OpenFIGI API key: {}", body));
|
||||||
|
} else if status == 413 {
|
||||||
|
return Err(anyhow::anyhow!("Payload too large—reduce chunk size: {}", body));
|
||||||
|
} else if status == 429 {
|
||||||
|
let reset = headers
|
||||||
|
.get("ratelimit-reset")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.unwrap_or("10")
|
||||||
|
.parse::<u64>()
|
||||||
|
.unwrap_or(10);
|
||||||
|
|
||||||
|
println!("Rate limited—backing off {}s", reset);
|
||||||
|
sleep(Duration::from_secs(reset.max(10))).await;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Err(anyhow::anyhow!("OpenFIGI error {}: {}", status, body));
|
||||||
|
}
|
||||||
|
|
||||||
|
// JSON aus dem *Body-String* parsen
|
||||||
|
let results: Vec<Value> = serde_json::from_str(&body)?;
|
||||||
|
for (job, result) in chunk.iter().zip(results) {
|
||||||
|
if let Some(data) = result["data"].as_array() {
|
||||||
|
for item in data {
|
||||||
|
let sec_type = item["securityType"].as_str().unwrap_or("");
|
||||||
|
let market_sec = item["marketSector"].as_str().unwrap_or("");
|
||||||
|
if market_sec == "Equity" &&
|
||||||
|
(sec_type.contains("Stock") || sec_type.contains("Share") || sec_type.contains("Equity") ||
|
||||||
|
sec_type.contains("Common") || sec_type.contains("Preferred") || sec_type == "ADR" || sec_type == "GDR") {
|
||||||
|
if let Some(figi) = item["figi"].as_str() {
|
||||||
|
all_figis.push(figi.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rate limit respect: 6s between requests with key
|
||||||
|
if self.has_key {
|
||||||
|
sleep(Duration::from_secs(6)).await;
|
||||||
|
} else {
|
||||||
|
sleep(Duration::from_millis(500)).await; // Slower without key
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
all_figis.dedup(); // Unique FIGIs per LEI
|
||||||
|
Ok(all_figis)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build FIGI → LEI map from CSV, filtering equities via OpenFIGI
|
||||||
|
pub async fn build_figi_to_lei_map(lei_to_isins: &HashMap<String, Vec<String>>) -> anyhow::Result<HashMap<String, String>> {
|
||||||
|
let client = OpenFigiClient::new()?;
|
||||||
|
if !client.has_key {
|
||||||
|
println!("No API key—skipping FIGI mapping (using empty map)");
|
||||||
|
return Ok(HashMap::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut figi_to_lei: HashMap<String, String> = HashMap::new();
|
||||||
|
let mut processed = 0;
|
||||||
|
|
||||||
|
for (lei, isins) in lei_to_isins {
|
||||||
|
let unique_isins: Vec<_> = isins.iter().cloned().collect::<HashSet<_>>().into_iter().collect();
|
||||||
|
let equity_figis = client.map_isins_to_figi(&unique_isins).await?;
|
||||||
|
|
||||||
|
for figi in equity_figis {
|
||||||
|
figi_to_lei.insert(figi, lei.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
processed += 1;
|
||||||
|
if processed % 100 == 0 {
|
||||||
|
println!("Processed {} LEIs → {} total equity FIGIs", processed, figi_to_lei.len());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Throttle per-LEI (heavy LEIs have 100s of ISINs)
|
||||||
|
sleep(Duration::from_millis(100)).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save full map
|
||||||
|
let data_dir = std::path::Path::new("data");
|
||||||
|
tokio::fs::create_dir_all(data_dir).await?;
|
||||||
|
tokio::fs::write("data/figi_to_lei.json", serde_json::to_string_pretty(&figi_to_lei)?).await?;
|
||||||
|
|
||||||
|
println!("Built FIGI→LEI map: {} mappings (equity-only)", figi_to_lei.len());
|
||||||
|
Ok(figi_to_lei)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Load/build companies using FIGI as key (enriched with LEI via map)
|
||||||
|
pub async fn load_or_build_companies_figi(
|
||||||
|
lei_to_isins: &HashMap<String, Vec<String>>,
|
||||||
|
figi_to_lei: &HashMap<String, String>,
|
||||||
|
) -> anyhow::Result<Vec<CompanyMetadata>> {
|
||||||
|
let data_dir = std::path::Path::new("data/companies_by_figi");
|
||||||
|
tokio::fs::create_dir_all(data_dir).await?;
|
||||||
|
|
||||||
|
let mut companies = Vec::new();
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
println!("Built {} FIGI-keyed companies.", companies.len());
|
||||||
|
Ok(companies)
|
||||||
|
}
|
||||||
@@ -1,16 +1,208 @@
|
|||||||
// src/corporate/scraper.rs
|
// src/corporate/scraper.rs
|
||||||
use super::types::{CompanyEvent, CompanyPrice};
|
use super::{types::*, helpers::*};
|
||||||
|
use csv::ReaderBuilder;
|
||||||
use fantoccini::{Client, Locator};
|
use fantoccini::{Client, Locator};
|
||||||
use scraper::{Html, Selector};
|
use scraper::{Html, Selector};
|
||||||
use chrono::{DateTime, Duration, NaiveDate, Timelike, Utc};
|
use chrono::{DateTime, Duration, NaiveDate, Timelike, Utc};
|
||||||
use tokio::time::{sleep, Duration as TokioDuration};
|
use tokio::{time::{Duration as TokioDuration, sleep}};
|
||||||
use reqwest::Client as HttpClient;
|
use reqwest::Client as HttpClient;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use yfinance_rs::{YfClient, Ticker, Range, Interval, HistoryBuilder};
|
use zip::ZipArchive;
|
||||||
use yfinance_rs::core::conversions::money_to_f64;
|
use std::fs::File;
|
||||||
|
use std::{collections::HashMap};
|
||||||
|
use std::io::{Read, BufReader};
|
||||||
|
|
||||||
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36";
|
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36";
|
||||||
|
|
||||||
|
/// Discover all exchanges where this ISIN trades by querying Yahoo Finance
|
||||||
|
pub async fn discover_available_exchanges(isin: &str, known_ticker: &str) -> anyhow::Result<Vec<TickerInfo>> {
|
||||||
|
println!(" Discovering exchanges for ISIN {}", isin);
|
||||||
|
|
||||||
|
let mut discovered_tickers = Vec::new();
|
||||||
|
|
||||||
|
// Try the primary ticker first
|
||||||
|
if let Ok(info) = check_ticker_exists(known_ticker).await {
|
||||||
|
discovered_tickers.push(info);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Search for ISIN directly on Yahoo to find other listings
|
||||||
|
let search_url = format!(
|
||||||
|
"https://query2.finance.yahoo.com/v1/finance/search?q={}"esCount=20&newsCount=0",
|
||||||
|
isin
|
||||||
|
);
|
||||||
|
|
||||||
|
match HttpClient::new()
|
||||||
|
.get(&search_url)
|
||||||
|
.header("User-Agent", USER_AGENT)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(resp) => {
|
||||||
|
if let Ok(json) = resp.json::<Value>().await {
|
||||||
|
if let Some(quotes) = json["quotes"].as_array() {
|
||||||
|
for quote in quotes {
|
||||||
|
// First: filter by quoteType directly from search results (faster rejection)
|
||||||
|
let quote_type = quote["quoteType"].as_str().unwrap_or("");
|
||||||
|
if quote_type.to_uppercase() != "EQUITY" {
|
||||||
|
continue; // Skip bonds, ETFs, mutual funds, options, etc.
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(symbol) = quote["symbol"].as_str() {
|
||||||
|
// Avoid duplicates
|
||||||
|
if discovered_tickers.iter().any(|t: &TickerInfo| t.ticker == symbol) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Double-check with full quote data (some search results are misleading)
|
||||||
|
match check_ticker_exists(symbol).await {
|
||||||
|
Ok(info) => {
|
||||||
|
println!(" Found equity listing: {} on {} ({})",
|
||||||
|
symbol, info.exchange_mic, info.currency);
|
||||||
|
discovered_tickers.push(info);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
// Most common: it's not actually equity or not tradable
|
||||||
|
// println!(" Rejected {}: {}", symbol, e);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Be respectful to Yahoo
|
||||||
|
sleep(TokioDuration::from_millis(120)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => println!(" Search API error: {}", e),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also try common exchange suffixes for the base ticker
|
||||||
|
if let Some(base) = known_ticker.split('.').next() {
|
||||||
|
let suffixes = vec![
|
||||||
|
"", // US
|
||||||
|
".L", // London
|
||||||
|
".DE", // Frankfurt/XETRA
|
||||||
|
".PA", // Paris
|
||||||
|
".AS", // Amsterdam
|
||||||
|
".MI", // Milan
|
||||||
|
".SW", // Switzerland
|
||||||
|
".T", // Tokyo
|
||||||
|
".HK", // Hong Kong
|
||||||
|
".SS", // Shanghai
|
||||||
|
".SZ", // Shenzhen
|
||||||
|
".TO", // Toronto
|
||||||
|
".AX", // Australia
|
||||||
|
".SA", // Brazil
|
||||||
|
".MC", // Madrid
|
||||||
|
".BO", // Bombay
|
||||||
|
".NS", // National Stock Exchange India
|
||||||
|
];
|
||||||
|
|
||||||
|
for suffix in suffixes {
|
||||||
|
let test_ticker = format!("{}{}", base, suffix);
|
||||||
|
|
||||||
|
// Skip if already found
|
||||||
|
if discovered_tickers.iter().any(|t| t.ticker == test_ticker) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Ok(info) = check_ticker_exists(&test_ticker).await {
|
||||||
|
discovered_tickers.push(info);
|
||||||
|
sleep(TokioDuration::from_millis(100)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
println!(" Found {} tradable exchanges", discovered_tickers.len());
|
||||||
|
Ok(discovered_tickers)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if a ticker exists and get its exchange/currency info
|
||||||
|
async fn check_ticker_exists(ticker: &str) -> anyhow::Result<TickerInfo> {
|
||||||
|
let url = format!(
|
||||||
|
"https://query1.finance.yahoo.com/v10/finance/quoteSummary/{}?modules=price",
|
||||||
|
ticker
|
||||||
|
);
|
||||||
|
|
||||||
|
let resp = HttpClient::new()
|
||||||
|
.get(&url)
|
||||||
|
.header("User-Agent", USER_AGENT)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let json: Value = resp.json().await?;
|
||||||
|
|
||||||
|
if let Some(result) = json["quoteSummary"]["result"].as_array() {
|
||||||
|
if result.is_empty() {
|
||||||
|
return Err(anyhow::anyhow!("No quote data for {}", ticker));
|
||||||
|
}
|
||||||
|
|
||||||
|
let quote = &result[0]["price"];
|
||||||
|
|
||||||
|
// CRITICAL: Only accept EQUITY securities
|
||||||
|
let quote_type = quote["quoteType"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_uppercase();
|
||||||
|
|
||||||
|
if quote_type != "EQUITY" {
|
||||||
|
// Optional: debug what was filtered
|
||||||
|
println!(" → Skipping {} (quoteType: {})", ticker, quote_type);
|
||||||
|
return Err(anyhow::anyhow!("Not an equity: {}", quote_type));
|
||||||
|
}
|
||||||
|
|
||||||
|
let exchange = quote["exchange"].as_str().unwrap_or("");
|
||||||
|
let currency = quote["currency"].as_str().unwrap_or("USD");
|
||||||
|
let short_name = quote["shortName"].as_str().unwrap_or("");
|
||||||
|
|
||||||
|
// Optional: extra sanity — make sure it's not a bond masquerading as equity
|
||||||
|
if short_name.to_uppercase().contains("BOND") ||
|
||||||
|
short_name.to_uppercase().contains("NOTE") ||
|
||||||
|
short_name.to_uppercase().contains("DEBENTURE") {
|
||||||
|
return Err(anyhow::anyhow!("Name suggests debt security"));
|
||||||
|
}
|
||||||
|
|
||||||
|
if !exchange.is_empty() {
|
||||||
|
return Ok(TickerInfo {
|
||||||
|
ticker: ticker.to_string(),
|
||||||
|
exchange_mic: exchange.to_string(),
|
||||||
|
currency: currency.to_string(),
|
||||||
|
primary: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(anyhow::anyhow!("Invalid or missing data for {}", ticker))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert Yahoo's exchange name to MIC code (best effort)
|
||||||
|
fn exchange_name_to_mic(name: &str) -> String {
|
||||||
|
match name {
|
||||||
|
"NMS" | "NasdaqGS" | "NASDAQ" => "XNAS",
|
||||||
|
"NYQ" | "NYSE" => "XNYS",
|
||||||
|
"LSE" | "London" => "XLON",
|
||||||
|
"FRA" | "Frankfurt" | "GER" | "XETRA" => "XFRA",
|
||||||
|
"PAR" | "Paris" => "XPAR",
|
||||||
|
"AMS" | "Amsterdam" => "XAMS",
|
||||||
|
"MIL" | "Milan" => "XMIL",
|
||||||
|
"JPX" | "Tokyo" => "XJPX",
|
||||||
|
"HKG" | "Hong Kong" => "XHKG",
|
||||||
|
"SHH" | "Shanghai" => "XSHG",
|
||||||
|
"SHZ" | "Shenzhen" => "XSHE",
|
||||||
|
"TOR" | "Toronto" => "XTSE",
|
||||||
|
"ASX" | "Australia" => "XASX",
|
||||||
|
"SAU" | "Saudi" => "XSAU",
|
||||||
|
"SWX" | "Switzerland" => "XSWX",
|
||||||
|
"BSE" | "Bombay" => "XBSE",
|
||||||
|
"NSE" | "NSI" => "XNSE",
|
||||||
|
"TAI" | "Taiwan" => "XTAI",
|
||||||
|
"SAO" | "Sao Paulo" => "BVMF",
|
||||||
|
"MCE" | "Madrid" => "XMAD",
|
||||||
|
_ => name, // Fallback to name itself
|
||||||
|
}.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn dismiss_yahoo_consent(client: &Client) -> anyhow::Result<()> {
|
pub async fn dismiss_yahoo_consent(client: &Client) -> anyhow::Result<()> {
|
||||||
let script = r#"
|
let script = r#"
|
||||||
(() => {
|
(() => {
|
||||||
@@ -34,14 +226,10 @@ pub async fn dismiss_yahoo_consent(client: &Client) -> anyhow::Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn fetch_earnings_history(client: &Client, ticker: &str) -> anyhow::Result<Vec<CompanyEvent>> {
|
pub async fn fetch_earnings_history(client: &Client, ticker: &str) -> anyhow::Result<Vec<CompanyEvent>> {
|
||||||
// Navigate to Yahoo Earnings Calendar for the ticker
|
|
||||||
// offset=0&size=100 to get up to 100 entries
|
|
||||||
// offset up to 99 loading older entries if needed
|
|
||||||
let url = format!("https://finance.yahoo.com/calendar/earnings?symbol={}&offset=0&size=100", ticker);
|
let url = format!("https://finance.yahoo.com/calendar/earnings?symbol={}&offset=0&size=100", ticker);
|
||||||
client.goto(&url).await?;
|
client.goto(&url).await?;
|
||||||
dismiss_yahoo_consent(client).await?;
|
dismiss_yahoo_consent(client).await?;
|
||||||
|
|
||||||
// Load all by clicking "Show More" if present (unchanged)
|
|
||||||
loop {
|
loop {
|
||||||
match client.find(Locator::XPath(r#"//button[contains(text(), 'Show More')]"#)).await {
|
match client.find(Locator::XPath(r#"//button[contains(text(), 'Show More')]"#)).await {
|
||||||
Ok(btn) => {
|
Ok(btn) => {
|
||||||
@@ -61,9 +249,9 @@ pub async fn fetch_earnings_history(client: &Client, ticker: &str) -> anyhow::Re
|
|||||||
let cols: Vec<String> = row.select(&Selector::parse("td").unwrap())
|
let cols: Vec<String> = row.select(&Selector::parse("td").unwrap())
|
||||||
.map(|td| td.text().collect::<Vec<_>>().join(" ").trim().to_string())
|
.map(|td| td.text().collect::<Vec<_>>().join(" ").trim().to_string())
|
||||||
.collect();
|
.collect();
|
||||||
if cols.len() < 6 { continue; } // Updated to match current 6-column structure
|
if cols.len() < 6 { continue; }
|
||||||
|
|
||||||
let full_date = &cols[2]; // Now Earnings Date
|
let full_date = &cols[2];
|
||||||
let parts: Vec<&str> = full_date.split(" at ").collect();
|
let parts: Vec<&str> = full_date.split(" at ").collect();
|
||||||
let raw_date = parts[0].trim();
|
let raw_date = parts[0].trim();
|
||||||
let time_str = if parts.len() > 1 { parts[1].trim() } else { "" };
|
let time_str = if parts.len() > 1 { parts[1].trim() } else { "" };
|
||||||
@@ -73,8 +261,8 @@ pub async fn fetch_earnings_history(client: &Client, ticker: &str) -> anyhow::Re
|
|||||||
Err(_) => continue,
|
Err(_) => continue,
|
||||||
};
|
};
|
||||||
|
|
||||||
let eps_forecast = parse_float(&cols[3]); // EPS Estimate
|
let eps_forecast = parse_float(&cols[3]);
|
||||||
let eps_actual = if cols[4] == "-" { None } else { parse_float(&cols[4]) }; // Reported EPS
|
let eps_actual = if cols[4] == "-" { None } else { parse_float(&cols[4]) };
|
||||||
|
|
||||||
let surprise_pct = if let (Some(f), Some(a)) = (eps_forecast, eps_actual) {
|
let surprise_pct = if let (Some(f), Some(a)) = (eps_forecast, eps_actual) {
|
||||||
if f.abs() > 0.001 { Some((a - f) / f.abs() * 100.0) } else { None }
|
if f.abs() > 0.001 { Some((a - f) / f.abs() * 100.0) } else { None }
|
||||||
@@ -105,7 +293,6 @@ pub async fn fetch_earnings_history(client: &Client, ticker: &str) -> anyhow::Re
|
|||||||
Ok(events)
|
Ok(events)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Helper: Yahoo returns prices as strings like "$123.45" or null
|
|
||||||
fn parse_price(v: Option<&Value>) -> f64 {
|
fn parse_price(v: Option<&Value>) -> f64 {
|
||||||
v.and_then(|x| x.as_str())
|
v.and_then(|x| x.as_str())
|
||||||
.and_then(|s| s.replace('$', "").replace(',', "").parse::<f64>().ok())
|
.and_then(|s| s.replace('$', "").replace(',', "").parse::<f64>().ok())
|
||||||
@@ -126,13 +313,13 @@ pub async fn fetch_daily_price_history(
|
|||||||
end_str: &str,
|
end_str: &str,
|
||||||
) -> anyhow::Result<Vec<CompanyPrice>> {
|
) -> anyhow::Result<Vec<CompanyPrice>> {
|
||||||
let start = NaiveDate::parse_from_str(start_str, "%Y-%m-%d")?;
|
let start = NaiveDate::parse_from_str(start_str, "%Y-%m-%d")?;
|
||||||
let end = NaiveDate::parse_from_str(end_str, "%Y-%m-%d")? + Duration::days(1); // inclusive
|
let end = NaiveDate::parse_from_str(end_str, "%Y-%m-%d")? + Duration::days(1);
|
||||||
|
|
||||||
let mut all_prices = Vec::new();
|
let mut all_prices = Vec::new();
|
||||||
let mut current = start;
|
let mut current = start;
|
||||||
|
|
||||||
while current < end {
|
while current < end {
|
||||||
let chunk_end = current + Duration::days(730); // 2-year chunks = safe
|
let chunk_end = current + Duration::days(730);
|
||||||
let actual_end = chunk_end.min(end);
|
let actual_end = chunk_end.min(end);
|
||||||
|
|
||||||
let period1 = current.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp();
|
let period1 = current.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp();
|
||||||
@@ -146,7 +333,7 @@ pub async fn fetch_daily_price_history(
|
|||||||
|
|
||||||
let json: Value = HttpClient::new()
|
let json: Value = HttpClient::new()
|
||||||
.get(&url)
|
.get(&url)
|
||||||
.header("User-Agent", "Mozilla/5.0")
|
.header("User-Agent", USER_AGENT)
|
||||||
.send()
|
.send()
|
||||||
.await?
|
.await?
|
||||||
.json()
|
.json()
|
||||||
@@ -155,12 +342,15 @@ pub async fn fetch_daily_price_history(
|
|||||||
let result = &json["chart"]["result"][0];
|
let result = &json["chart"]["result"][0];
|
||||||
let timestamps = result["timestamp"].as_array().ok_or_else(|| anyhow::anyhow!("No timestamps"))?;
|
let timestamps = result["timestamp"].as_array().ok_or_else(|| anyhow::anyhow!("No timestamps"))?;
|
||||||
let quote = &result["indicators"]["quote"][0];
|
let quote = &result["indicators"]["quote"][0];
|
||||||
|
let meta = &result["meta"];
|
||||||
|
let currency = meta["currency"].as_str().unwrap_or("USD").to_string();
|
||||||
|
|
||||||
let opens = quote["open"].as_array();
|
let opens = quote["open"].as_array();
|
||||||
let highs = quote["high"].as_array();
|
let highs = quote["high"].as_array();
|
||||||
let lows = quote["low"].as_array();
|
let lows = quote["low"].as_array();
|
||||||
let closes = quote["close"].as_array();
|
let closes = quote["close"].as_array();
|
||||||
let adj_closes = result["meta"]["adjClose"].as_array().or_else(|| quote["close"].as_array()); // fallback
|
let adj_closes = result["indicators"]["adjclose"][0]["adjclose"].as_array()
|
||||||
|
.or_else(|| closes);
|
||||||
let volumes = quote["volume"].as_array();
|
let volumes = quote["volume"].as_array();
|
||||||
|
|
||||||
for (i, ts_val) in timestamps.iter().enumerate() {
|
for (i, ts_val) in timestamps.iter().enumerate() {
|
||||||
@@ -182,21 +372,23 @@ pub async fn fetch_daily_price_history(
|
|||||||
all_prices.push(CompanyPrice {
|
all_prices.push(CompanyPrice {
|
||||||
ticker: ticker.to_string(),
|
ticker: ticker.to_string(),
|
||||||
date: date_str,
|
date: date_str,
|
||||||
|
time: "".to_string(),
|
||||||
open,
|
open,
|
||||||
high,
|
high,
|
||||||
low,
|
low,
|
||||||
close,
|
close,
|
||||||
adj_close,
|
adj_close,
|
||||||
volume,
|
volume,
|
||||||
|
currency: currency.clone(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
sleep(TokioDuration::from_millis(200));
|
sleep(TokioDuration::from_millis(200)).await;
|
||||||
current = actual_end;
|
current = actual_end;
|
||||||
}
|
}
|
||||||
|
|
||||||
all_prices.sort_by_key(|p| p.date.clone());
|
all_prices.sort_by_key(|p| (p.date.clone(), p.time.clone()));
|
||||||
all_prices.dedup_by_key(|p| p.date.clone());
|
all_prices.dedup_by(|a, b| a.date == b.date && a.time == b.time);
|
||||||
|
|
||||||
println!(" Got {} daily bars for {ticker}", all_prices.len());
|
println!(" Got {} daily bars for {ticker}", all_prices.len());
|
||||||
Ok(all_prices)
|
Ok(all_prices)
|
||||||
@@ -207,8 +399,8 @@ pub async fn fetch_price_history_5min(
|
|||||||
_start: &str,
|
_start: &str,
|
||||||
_end: &str,
|
_end: &str,
|
||||||
) -> anyhow::Result<Vec<CompanyPrice>> {
|
) -> anyhow::Result<Vec<CompanyPrice>> {
|
||||||
let now = Utc::now().timestamp();
|
let now = Utc::now().timestamp();
|
||||||
let period1 = now - 5184000; // 60 days ago
|
let period1 = now - 5184000;
|
||||||
let period2 = now;
|
let period2 = now;
|
||||||
|
|
||||||
let url = format!(
|
let url = format!(
|
||||||
@@ -217,7 +409,7 @@ let now = Utc::now().timestamp();
|
|||||||
|
|
||||||
let json: Value = HttpClient::new()
|
let json: Value = HttpClient::new()
|
||||||
.get(&url)
|
.get(&url)
|
||||||
.header("User-Agent", "Mozilla/5.0")
|
.header("User-Agent", USER_AGENT)
|
||||||
.send()
|
.send()
|
||||||
.await?
|
.await?
|
||||||
.json()
|
.json()
|
||||||
@@ -226,6 +418,8 @@ let now = Utc::now().timestamp();
|
|||||||
let result = &json["chart"]["result"][0];
|
let result = &json["chart"]["result"][0];
|
||||||
let timestamps = result["timestamp"].as_array().ok_or_else(|| anyhow::anyhow!("No timestamps"))?;
|
let timestamps = result["timestamp"].as_array().ok_or_else(|| anyhow::anyhow!("No timestamps"))?;
|
||||||
let quote = &result["indicators"]["quote"][0];
|
let quote = &result["indicators"]["quote"][0];
|
||||||
|
let meta = &result["meta"];
|
||||||
|
let currency = meta["currency"].as_str().unwrap_or("USD").to_string();
|
||||||
|
|
||||||
let mut prices = Vec::new();
|
let mut prices = Vec::new();
|
||||||
|
|
||||||
@@ -233,6 +427,7 @@ let now = Utc::now().timestamp();
|
|||||||
let ts = ts_val.as_i64().unwrap_or(0);
|
let ts = ts_val.as_i64().unwrap_or(0);
|
||||||
let dt: DateTime<Utc> = DateTime::from_timestamp(ts, 0).unwrap_or_default();
|
let dt: DateTime<Utc> = DateTime::from_timestamp(ts, 0).unwrap_or_default();
|
||||||
let date_str = dt.format("%Y-%m-%d").to_string();
|
let date_str = dt.format("%Y-%m-%d").to_string();
|
||||||
|
let time_str = dt.format("%H:%M:%S").to_string();
|
||||||
|
|
||||||
let open = parse_price(quote["open"].as_array().and_then(|a| a.get(i)));
|
let open = parse_price(quote["open"].as_array().and_then(|a| a.get(i)));
|
||||||
let high = parse_price(quote["high"].as_array().and_then(|a| a.get(i)));
|
let high = parse_price(quote["high"].as_array().and_then(|a| a.get(i)));
|
||||||
@@ -243,25 +438,254 @@ let now = Utc::now().timestamp();
|
|||||||
prices.push(CompanyPrice {
|
prices.push(CompanyPrice {
|
||||||
ticker: ticker.to_string(),
|
ticker: ticker.to_string(),
|
||||||
date: date_str,
|
date: date_str,
|
||||||
|
time: time_str,
|
||||||
open,
|
open,
|
||||||
high,
|
high,
|
||||||
low,
|
low,
|
||||||
close,
|
close,
|
||||||
adj_close: close, // intraday usually not adjusted
|
adj_close: close,
|
||||||
volume,
|
volume,
|
||||||
|
currency: currency.clone(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
prices.sort_by_key(|p| p.date.clone());
|
prices.sort_by_key(|p| (p.date.clone(), p.time.clone()));
|
||||||
Ok(prices)
|
Ok(prices)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_float(s: &str) -> Option<f64> {
|
/// Fetch the URL of the latest ISIN↔LEI mapping CSV from GLEIF
|
||||||
s.replace("--", "").replace(",", "").parse::<f64>().ok()
|
/// Overengineered; we could just use the static URL, but this shows how to scrape if needed
|
||||||
|
pub async fn _fetch_latest_gleif_isin_lei_mapping_url(client: &Client) -> anyhow::Result<String> {
|
||||||
|
let url = format!("https://www.gleif.org/de/lei-data/lei-mapping/download-isin-to-lei-relationship-files");
|
||||||
|
client.goto(&url).await?;
|
||||||
|
|
||||||
|
let html = client.source().await?;
|
||||||
|
let _document = Html::parse_document(&html);
|
||||||
|
let _row_sel = Selector::parse("table tbody tr").unwrap();
|
||||||
|
let isin_lei = "".to_string();
|
||||||
|
|
||||||
|
Ok(isin_lei)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_yahoo_date(s: &str) -> anyhow::Result<NaiveDate> {
|
pub async fn download_isin_lei_csv() -> anyhow::Result<Option<String>> {
|
||||||
NaiveDate::parse_from_str(s, "%B %d, %Y")
|
let url = "https://mapping.gleif.org/api/v2/isin-lei/9315e3e3-305a-4e71-b062-46714740fa8d/download";
|
||||||
.or_else(|_| NaiveDate::parse_from_str(s, "%b %d, %Y"))
|
let zip_path = "data/isin_lei.zip";
|
||||||
.map_err(|_| anyhow::anyhow!("Bad date: {s}"))
|
let csv_path = "data/isin_lei.csv";
|
||||||
|
|
||||||
|
if let Err(e) = std::fs::create_dir_all("data") {
|
||||||
|
println!("Failed to create data directory: {e}");
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download ZIP
|
||||||
|
let bytes = match reqwest::Client::builder()
|
||||||
|
.user_agent(USER_AGENT)
|
||||||
|
.timeout(std::time::Duration::from_secs(30))
|
||||||
|
.build()
|
||||||
|
.and_then(|c| Ok(c))
|
||||||
|
{
|
||||||
|
Ok(client) => match client.get(url).send().await {
|
||||||
|
Ok(resp) if resp.status().is_success() => match resp.bytes().await {
|
||||||
|
Ok(b) => b,
|
||||||
|
Err(e) => {
|
||||||
|
println!("Failed to read ZIP bytes: {e}");
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Ok(resp) => {
|
||||||
|
println!("Server returned HTTP {}", resp.status());
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
println!("Failed to download ISIN/LEI ZIP: {e}");
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(e) => {
|
||||||
|
println!("Failed to create HTTP client: {e}");
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Err(e) = tokio::fs::write(zip_path, &bytes).await {
|
||||||
|
println!("Failed to write ZIP file: {e}");
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract CSV
|
||||||
|
let archive = match std::fs::File::open(zip_path)
|
||||||
|
.map(ZipArchive::new)
|
||||||
|
{
|
||||||
|
Ok(Ok(a)) => a,
|
||||||
|
Ok(Err(e)) => {
|
||||||
|
println!("Invalid ZIP: {e}");
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
println!("Cannot open ZIP file: {e}");
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut archive = archive;
|
||||||
|
|
||||||
|
let idx = match (0..archive.len()).find(|&i| {
|
||||||
|
archive.by_index(i)
|
||||||
|
.map(|f| f.name().ends_with(".csv"))
|
||||||
|
.unwrap_or(false)
|
||||||
|
}) {
|
||||||
|
Some(i) => i,
|
||||||
|
None => {
|
||||||
|
println!("ZIP did not contain a CSV file");
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut csv_file = match archive.by_index(idx) {
|
||||||
|
Ok(f) => f,
|
||||||
|
Err(e) => {
|
||||||
|
println!("Failed to read CSV entry: {e}");
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut csv_bytes = Vec::new();
|
||||||
|
if let Err(e) = csv_file.read_to_end(&mut csv_bytes) {
|
||||||
|
println!("Failed to extract CSV: {e}");
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Err(e) = tokio::fs::write(csv_path, &csv_bytes).await {
|
||||||
|
println!("Failed to save CSV file: {e}");
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Some(csv_path.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
pub async fn load_isin_lei_csv() -> anyhow::Result<HashMap<String, Vec<String>>> {
|
||||||
|
// 1. Download + extract the CSV (this is now async)
|
||||||
|
let csv_path = match download_isin_lei_csv().await? {
|
||||||
|
Some(p) => p,
|
||||||
|
None => {
|
||||||
|
println!("ISIN/LEI download failed; continuing with empty map");
|
||||||
|
return Ok(HashMap::new());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// 2. Open and parse the CSV synchronously (fast enough, ~8M lines is fine)
|
||||||
|
let file = match std::fs::File::open(&csv_path) {
|
||||||
|
Ok(f) => f,
|
||||||
|
Err(e) => {
|
||||||
|
println!("Cannot open CSV '{}': {}", csv_path, e);
|
||||||
|
return Ok(HashMap::new());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut rdr = csv::ReaderBuilder::new()
|
||||||
|
.has_headers(false)
|
||||||
|
.from_reader(std::io::BufReader::new(file));
|
||||||
|
|
||||||
|
let mut map: HashMap<String, Vec<String>> = HashMap::new();
|
||||||
|
|
||||||
|
for result in rdr.records() {
|
||||||
|
let record = match result {
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(e) => {
|
||||||
|
println!("CSV parse error: {}", e);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if record.len() < 2 { continue; }
|
||||||
|
|
||||||
|
let lei = record[0].to_string();
|
||||||
|
let isin = record[1].to_string();
|
||||||
|
map.entry(lei).or_default().push(isin);
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("Loaded ISIN↔LEI map with {} LEIs and {} total ISINs",
|
||||||
|
map.len(),
|
||||||
|
map.values().map(|v| v.len()).sum::<usize>()
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(map)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get_primary_isin_and_name(
|
||||||
|
client: &Client, // Pass your existing Selenium client
|
||||||
|
ticker: &str,
|
||||||
|
) -> anyhow::Result<PrimaryInfo> {
|
||||||
|
// Navigate to the actual quote page (always works)
|
||||||
|
let quote_url = format!("https://finance.yahoo.com/quote/{}", ticker);
|
||||||
|
client.goto("e_url).await?;
|
||||||
|
|
||||||
|
// Dismiss overlays/banners (your function + guce-specific)
|
||||||
|
reject_yahoo_cookies(client).await?;
|
||||||
|
|
||||||
|
// Wait for page to load (key data elements)
|
||||||
|
sleep(TokioDuration::from_millis(2000)).await;
|
||||||
|
|
||||||
|
// Get page HTML and parse
|
||||||
|
let html = client.source().await?;
|
||||||
|
let document = Html::parse_document(&html);
|
||||||
|
|
||||||
|
// Selectors for key fields (tested on real Yahoo pages Nov 2025)
|
||||||
|
let name_sel = Selector::parse("h1[data-testid='qsp-price-header']").unwrap_or_else(|_| Selector::parse("h1").unwrap());
|
||||||
|
let isin_sel = Selector::parse("[data-testid='qsp-symbol'] + div [data-field='isin']").unwrap_or_else(|_| Selector::parse("[data-field='isin']").unwrap());
|
||||||
|
let exchange_sel = Selector::parse("[data-testid='qsp-market'] span").unwrap_or_else(|_| Selector::parse(".TopNav__Exchange").unwrap());
|
||||||
|
let currency_sel = Selector::parse("[data-testid='qsp-price'] span:contains('USD')").unwrap_or_else(|_| Selector::parse(".TopNav__Currency").unwrap()); // Adjust for dynamic
|
||||||
|
|
||||||
|
let name_elem = document.select(&name_sel).next().map(|e| e.text().collect::<String>().trim().to_string());
|
||||||
|
let isin_elem = document.select(&isin_sel).next().map(|e| e.text().collect::<String>().trim().to_uppercase());
|
||||||
|
let exchange_elem = document.select(&exchange_sel).next().map(|e| e.text().collect::<String>().trim().to_string());
|
||||||
|
let currency_elem = document.select(¤cy_sel).next().map(|e| e.text().collect::<String>().trim().to_string());
|
||||||
|
|
||||||
|
let name = name_elem.unwrap_or_else(|| ticker.to_string());
|
||||||
|
let isin = isin_elem.unwrap_or_default();
|
||||||
|
let exchange_mic = exchange_elem.unwrap_or_default();
|
||||||
|
let currency = currency_elem.unwrap_or_else(|| "USD".to_string());
|
||||||
|
|
||||||
|
// Validate ISIN
|
||||||
|
let valid_isin = if isin.len() == 12 && isin.chars().all(|c| c.is_alphanumeric()) {
|
||||||
|
isin
|
||||||
|
} else {
|
||||||
|
"".to_string()
|
||||||
|
};
|
||||||
|
|
||||||
|
println!(" → Scraped {}: {} | ISIN: {} | Exchange: {}", ticker, name, valid_isin, exchange_mic);
|
||||||
|
|
||||||
|
Ok(PrimaryInfo {
|
||||||
|
isin: valid_isin,
|
||||||
|
name,
|
||||||
|
exchange_mic,
|
||||||
|
currency,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn reject_yahoo_cookies(client: &Client) -> anyhow::Result<()> {
|
||||||
|
for _ in 0..10 {
|
||||||
|
let clicked: bool = client
|
||||||
|
.execute(
|
||||||
|
r#"(() => {
|
||||||
|
const btn = document.querySelector('#consent-page .reject-all');
|
||||||
|
if (btn) {
|
||||||
|
btn.click();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
})()"#,
|
||||||
|
vec![],
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
.as_bool()
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
|
if clicked { break; }
|
||||||
|
sleep(TokioDuration::from_millis(500)).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("Rejected Yahoo cookies if button existed");
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1,9 +1,11 @@
|
|||||||
// src/corporate/storage.rs
|
// src/corporate/storage.rs
|
||||||
use super::types::{CompanyEvent, CompanyPrice, CompanyEventChange};
|
use super::{types::*, helpers::*, scraper::get_primary_isin_and_name};
|
||||||
use super::helpers::*;
|
use crate::config;
|
||||||
|
|
||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
use chrono::{Datelike, NaiveDate};
|
use chrono::{Datelike, NaiveDate};
|
||||||
use std::collections::HashMap;
|
use std::collections::{HashMap, HashSet};
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
pub async fn load_existing_events() -> anyhow::Result<HashMap<String, CompanyEvent>> {
|
pub async fn load_existing_events() -> anyhow::Result<HashMap<String, CompanyEvent>> {
|
||||||
let mut map = HashMap::new();
|
let mut map = HashMap::new();
|
||||||
@@ -17,7 +19,7 @@ pub async fn load_existing_events() -> anyhow::Result<HashMap<String, CompanyEve
|
|||||||
let path = entry.path();
|
let path = entry.path();
|
||||||
if path.extension().and_then(|s| s.to_str()) == Some("json") {
|
if path.extension().and_then(|s| s.to_str()) == Some("json") {
|
||||||
let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
|
let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
|
||||||
if name.starts_with("events_") && name.len() == 17 { // events_yyyy-mm.json
|
if name.starts_with("events_") && name.len() == 17 {
|
||||||
let content = fs::read_to_string(&path).await?;
|
let content = fs::read_to_string(&path).await?;
|
||||||
let events: Vec<CompanyEvent> = serde_json::from_str(&content)?;
|
let events: Vec<CompanyEvent> = serde_json::from_str(&content)?;
|
||||||
for event in events {
|
for event in events {
|
||||||
@@ -33,7 +35,6 @@ pub async fn save_optimized_events(events: HashMap<String, CompanyEvent>) -> any
|
|||||||
let dir = std::path::Path::new("corporate_events");
|
let dir = std::path::Path::new("corporate_events");
|
||||||
fs::create_dir_all(dir).await?;
|
fs::create_dir_all(dir).await?;
|
||||||
|
|
||||||
// Delete old files
|
|
||||||
let mut entries = fs::read_dir(dir).await?;
|
let mut entries = fs::read_dir(dir).await?;
|
||||||
while let Some(entry) = entries.next_entry().await? {
|
while let Some(entry) = entries.next_entry().await? {
|
||||||
let path = entry.path();
|
let path = entry.path();
|
||||||
@@ -87,13 +88,165 @@ pub async fn save_changes(changes: &[CompanyEventChange]) -> anyhow::Result<()>
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn save_prices_for_ticker(ticker: &str, timeframe: &str, mut prices: Vec<CompanyPrice>) -> anyhow::Result<()> {
|
pub async fn save_prices_for_ticker(ticker: &str, timeframe: &str, mut prices: Vec<CompanyPrice>) -> anyhow::Result<()> {
|
||||||
let dir = std::path::Path::new("corporate_prices");
|
let base_dir = Path::new("corporate_prices");
|
||||||
fs::create_dir_all(dir).await?;
|
let company_dir = base_dir.join(ticker.replace(".", "_"));
|
||||||
let path = dir.join(format!("{}_{}.json", ticker.replace(".", "_"), timeframe));
|
let timeframe_dir = company_dir.join(timeframe);
|
||||||
|
|
||||||
prices.sort_by_key(|p| p.date.clone());
|
fs::create_dir_all(&timeframe_dir).await?;
|
||||||
|
let path = timeframe_dir.join("prices.json");
|
||||||
|
|
||||||
|
prices.sort_by_key(|p| (p.date.clone(), p.time.clone()));
|
||||||
|
|
||||||
let json = serde_json::to_string_pretty(&prices)?;
|
let json = serde_json::to_string_pretty(&prices)?;
|
||||||
fs::write(&path, json).await?;
|
fs::write(&path, json).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn _load_companies() -> Result<Vec<CompanyMetadata>, anyhow::Error> {
|
||||||
|
let path = Path::new("src/data/companies.json");
|
||||||
|
if !path.exists() {
|
||||||
|
println!("Missing companies.json file at src/data/companies.json");
|
||||||
|
return Ok(vec![]);
|
||||||
|
}
|
||||||
|
let content = fs::read_to_string(path).await?;
|
||||||
|
let companies: Vec<CompanyMetadata> = serde_json::from_str(&content)?;
|
||||||
|
Ok(companies)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_company_dir(lei: &str) -> PathBuf {
|
||||||
|
PathBuf::from("corporate_prices").join(lei)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn ensure_company_dirs(isin: &str) -> anyhow::Result<()> {
|
||||||
|
let base = get_company_dir(isin);
|
||||||
|
let paths = [
|
||||||
|
base.clone(),
|
||||||
|
base.join("5min"),
|
||||||
|
base.join("daily"),
|
||||||
|
base.join("aggregated").join("5min"),
|
||||||
|
base.join("aggregated").join("daily"),
|
||||||
|
];
|
||||||
|
for p in paths {
|
||||||
|
fs::create_dir_all(&p).await?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn save_company_metadata(company: &CompanyMetadata) -> anyhow::Result<()> {
|
||||||
|
let dir = get_company_dir(&company.lei);
|
||||||
|
fs::create_dir_all(&dir).await?;
|
||||||
|
let path = dir.join("metadata.json");
|
||||||
|
fs::write(&path, serde_json::to_string_pretty(company)?).await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn load_company_metadata(lei: &str) -> anyhow::Result<CompanyMetadata> {
|
||||||
|
let path = get_company_dir(lei).join("metadata.json");
|
||||||
|
let content = fs::read_to_string(path).await?;
|
||||||
|
Ok(serde_json::from_str(&content)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn save_available_exchanges(isin: &str, exchanges: Vec<AvailableExchange>) -> anyhow::Result<()> {
|
||||||
|
let dir = get_company_dir(isin);
|
||||||
|
fs::create_dir_all(&dir).await?;
|
||||||
|
let path = dir.join("available_exchanges.json");
|
||||||
|
fs::write(&path, serde_json::to_string_pretty(&exchanges)?).await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn load_available_exchanges(lei: &str) -> anyhow::Result<Vec<AvailableExchange>> {
|
||||||
|
let path = get_company_dir(lei).join("available_exchanges.json");
|
||||||
|
if path.exists() {
|
||||||
|
let content = fs::read_to_string(&path).await?;
|
||||||
|
Ok(serde_json::from_str(&content)?)
|
||||||
|
} else {
|
||||||
|
Ok(vec![])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn save_prices_by_source(
|
||||||
|
lei: &str,
|
||||||
|
source_ticker: &str,
|
||||||
|
timeframe: &str,
|
||||||
|
prices: Vec<CompanyPrice>,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let source_safe = source_ticker.replace(".", "_").replace("/", "_");
|
||||||
|
let dir = get_company_dir(lei).join(timeframe).join(&source_safe);
|
||||||
|
fs::create_dir_all(&dir).await?;
|
||||||
|
let path = dir.join("prices.json");
|
||||||
|
let mut prices = prices;
|
||||||
|
prices.sort_by_key(|p| (p.date.clone(), p.time.clone()));
|
||||||
|
fs::write(&path, serde_json::to_string_pretty(&prices)?).await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Update available_exchanges.json with fetch results
|
||||||
|
pub async fn update_available_exchange(
|
||||||
|
isin: &str,
|
||||||
|
ticker: &str,
|
||||||
|
exchange_mic: &str,
|
||||||
|
has_daily: bool,
|
||||||
|
has_5min: bool,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let mut exchanges = load_available_exchanges(isin).await?;
|
||||||
|
|
||||||
|
if let Some(entry) = exchanges.iter_mut().find(|e| e.ticker == ticker) {
|
||||||
|
// Update existing entry
|
||||||
|
entry.record_success(has_daily, has_5min);
|
||||||
|
} else {
|
||||||
|
// Create new entry - need to get currency from somewhere
|
||||||
|
// Try to infer from the ticker or use a default
|
||||||
|
let currency = infer_currency_from_ticker(ticker);
|
||||||
|
let mut new_entry = AvailableExchange::new(
|
||||||
|
ticker.to_string(),
|
||||||
|
exchange_mic.to_string(),
|
||||||
|
currency,
|
||||||
|
);
|
||||||
|
new_entry.record_success(has_daily, has_5min);
|
||||||
|
exchanges.push(new_entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
save_available_exchanges(isin, exchanges).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Add a newly discovered exchange before fetching
|
||||||
|
pub async fn add_discovered_exchange(
|
||||||
|
isin: &str,
|
||||||
|
ticker_info: &TickerInfo,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let mut exchanges = load_available_exchanges(isin).await?;
|
||||||
|
|
||||||
|
// Only add if not already present
|
||||||
|
if !exchanges.iter().any(|e| e.ticker == ticker_info.ticker) {
|
||||||
|
let new_entry = AvailableExchange::new(
|
||||||
|
ticker_info.ticker.clone(),
|
||||||
|
ticker_info.exchange_mic.clone(),
|
||||||
|
ticker_info.currency.clone(),
|
||||||
|
);
|
||||||
|
exchanges.push(new_entry);
|
||||||
|
save_available_exchanges(isin, exchanges).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Infer currency from ticker suffix
|
||||||
|
fn infer_currency_from_ticker(ticker: &str) -> String {
|
||||||
|
if ticker.ends_with(".L") { return "GBP".to_string(); }
|
||||||
|
if ticker.ends_with(".PA") { return "EUR".to_string(); }
|
||||||
|
if ticker.ends_with(".DE") { return "EUR".to_string(); }
|
||||||
|
if ticker.ends_with(".AS") { return "EUR".to_string(); }
|
||||||
|
if ticker.ends_with(".MI") { return "EUR".to_string(); }
|
||||||
|
if ticker.ends_with(".SW") { return "CHF".to_string(); }
|
||||||
|
if ticker.ends_with(".T") { return "JPY".to_string(); }
|
||||||
|
if ticker.ends_with(".HK") { return "HKD".to_string(); }
|
||||||
|
if ticker.ends_with(".SS") { return "CNY".to_string(); }
|
||||||
|
if ticker.ends_with(".SZ") { return "CNY".to_string(); }
|
||||||
|
if ticker.ends_with(".TO") { return "CAD".to_string(); }
|
||||||
|
if ticker.ends_with(".AX") { return "AUD".to_string(); }
|
||||||
|
if ticker.ends_with(".SA") { return "BRL".to_string(); }
|
||||||
|
if ticker.ends_with(".MC") { return "EUR".to_string(); }
|
||||||
|
if ticker.ends_with(".BO") || ticker.ends_with(".NS") { return "INR".to_string(); }
|
||||||
|
|
||||||
|
"USD".to_string() // Default
|
||||||
|
}
|
||||||
@@ -19,20 +19,86 @@ pub struct CompanyEvent {
|
|||||||
pub struct CompanyPrice {
|
pub struct CompanyPrice {
|
||||||
pub ticker: String,
|
pub ticker: String,
|
||||||
pub date: String, // YYYY-MM-DD
|
pub date: String, // YYYY-MM-DD
|
||||||
|
pub time: String, // HH:MM:SS for intraday, "" for daily
|
||||||
pub open: f64,
|
pub open: f64,
|
||||||
pub high: f64,
|
pub high: f64,
|
||||||
pub low: f64,
|
pub low: f64,
|
||||||
pub close: f64,
|
pub close: f64,
|
||||||
pub adj_close: f64,
|
pub adj_close: f64,
|
||||||
pub volume: u64,
|
pub volume: u64,
|
||||||
|
pub currency: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct CompanyEventChange {
|
pub struct CompanyEventChange {
|
||||||
pub ticker: String,
|
pub ticker: String,
|
||||||
pub date: String,
|
pub date: String,
|
||||||
pub field_changed: String, // "time", "eps_forecast", "eps_actual", "new_event"
|
pub field_changed: String, // "time", "eps_forecast", "eps_actual", "new_event"
|
||||||
pub old_value: String,
|
pub old_value: String,
|
||||||
pub new_value: String,
|
pub new_value: String,
|
||||||
pub detected_at: String,
|
pub detected_at: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct TickerInfo {
|
||||||
|
pub ticker: String,
|
||||||
|
pub exchange_mic: String,
|
||||||
|
pub currency: String,
|
||||||
|
pub isin: String, // ISIN belonging to this legal entity (primary + ADR + GDR)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct CompanyMetadata {
|
||||||
|
pub lei: String,
|
||||||
|
pub figi: Option<String>,
|
||||||
|
pub name: String,
|
||||||
|
pub primary_isin: String, // The most liquid / preferred one (used for folder fallback)
|
||||||
|
pub tickers: Vec<TickerInfo>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct PrimaryInfo {
|
||||||
|
pub isin: String,
|
||||||
|
pub name: String,
|
||||||
|
pub exchange_mic: String,
|
||||||
|
pub currency: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct AvailableExchange {
|
||||||
|
pub exchange_mic: String,
|
||||||
|
pub ticker: String,
|
||||||
|
pub has_daily: bool,
|
||||||
|
pub has_5min: bool,
|
||||||
|
pub last_successful_fetch: Option<String>, // YYYY-MM-DD
|
||||||
|
#[serde(default)]
|
||||||
|
pub currency: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub discovered_at: Option<String>, // When this exchange was first discovered
|
||||||
|
#[serde(default)]
|
||||||
|
pub fetch_count: u32, // How many times successfully fetched
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AvailableExchange {
|
||||||
|
pub fn new(ticker: String, exchange_mic: String, currency: String) -> Self {
|
||||||
|
Self {
|
||||||
|
exchange_mic,
|
||||||
|
ticker,
|
||||||
|
has_daily: false,
|
||||||
|
has_5min: false,
|
||||||
|
last_successful_fetch: None,
|
||||||
|
currency,
|
||||||
|
discovered_at: Some(chrono::Local::now().format("%Y-%m-%d").to_string()),
|
||||||
|
fetch_count: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_success(&mut self, has_daily: bool, has_5min: bool) {
|
||||||
|
let today = chrono::Local::now().format("%Y-%m-%d").to_string();
|
||||||
|
|
||||||
|
self.has_daily |= has_daily;
|
||||||
|
self.has_5min |= has_5min;
|
||||||
|
self.last_successful_fetch = Some(today);
|
||||||
|
self.fetch_count += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,57 +1,187 @@
|
|||||||
// src/corporate/update.rs
|
// src/corporate/update.rs
|
||||||
use super::{scraper::*, storage::*, helpers::*, types::*};
|
use super::{scraper::*, storage::*, helpers::*, types::*, aggregation::*, openfigi::*};
|
||||||
use crate::config::Config;
|
use crate::config::Config;
|
||||||
use yfinance_rs::{Range, Interval};
|
|
||||||
|
|
||||||
use chrono::Local;
|
use chrono::Local;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
pub async fn run_full_update(client: &fantoccini::Client, tickers: Vec<String>, config: &Config) -> anyhow::Result<()> {
|
pub async fn run_full_update(client: &fantoccini::Client, config: &Config) -> anyhow::Result<()> {
|
||||||
println!("Updating {} tickers (prices from {})", tickers.len(), config.corporate_start_date);
|
println!("Starting LEI-based corporate update");
|
||||||
|
|
||||||
|
// 1. Download fresh GLEIF ISIN↔LEI mapping on every run
|
||||||
|
let lei_to_isins: HashMap<String, Vec<String>> = match load_isin_lei_csv().await {
|
||||||
|
Ok(map) => map,
|
||||||
|
Err(e) => {
|
||||||
|
println!("Warning: Failed to load ISIN↔LEI mapping: {}", e);
|
||||||
|
HashMap::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let figi_to_lei: HashMap<String, String> = match build_figi_to_lei_map(&lei_to_isins).await {
|
||||||
|
Ok(map) => map,
|
||||||
|
Err(e) => {
|
||||||
|
println!("Warning: Failed to build FIGI→LEI map: {}", e);
|
||||||
|
HashMap::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
let today = chrono::Local::now().format("%Y-%m-%d").to_string();
|
let today = chrono::Local::now().format("%Y-%m-%d").to_string();
|
||||||
|
let mut existing_events = load_existing_events().await?;
|
||||||
|
|
||||||
let mut existing = load_existing_events().await?;
|
let mut companies: Vec<CompanyMetadata> = match load_or_build_companies_figi(&lei_to_isins, &figi_to_lei).await {
|
||||||
|
Ok(comps) => comps,
|
||||||
for ticker in &tickers {
|
Err(e) => {
|
||||||
print!(" → {:6} ", ticker);
|
println!("Error loading/building company metadata: {}", e);
|
||||||
|
return Err(e);
|
||||||
if let Ok(new_events) = fetch_earnings_history(client, ticker).await {
|
|
||||||
let result = process_batch(&new_events, &mut existing, &today);
|
|
||||||
save_changes(&result.changes).await?;
|
|
||||||
println!("{} earnings, {} changes", new_events.len(), result.changes.len());
|
|
||||||
}
|
}
|
||||||
|
}; // Vec<CompanyMetadata> with lei, isins, tickers
|
||||||
|
|
||||||
// DAILY – full history
|
for mut company in companies {
|
||||||
if let Ok(prices) = fetch_daily_price_history(ticker, &config.corporate_start_date, &today).await {
|
println!("\nProcessing company: {} (LEI: {})", company.name, company.lei);
|
||||||
save_prices_for_ticker(ticker, "daily", prices).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;
|
// === Enrich with ALL ISINs known to GLEIF (includes ADRs, GDRs, etc.) ===
|
||||||
|
if let Some(all_isins) = lei_to_isins.get(&company.lei) {
|
||||||
// 5-MINUTE – only last 60 days (Yahoo limit for intraday)
|
let mut seen = company.isins.iter().cloned().collect::<std::collections::HashSet<_>>();
|
||||||
let sixty_days_ago = (chrono::Local::now() - chrono::Duration::days(60))
|
for isin in all_isins {
|
||||||
.format("%Y-%m-%d")
|
if !seen.contains(isin) {
|
||||||
.to_string();
|
company.isins.push(isin.clone());
|
||||||
|
seen.insert(isin.clone());
|
||||||
if let Ok(prices) = fetch_price_history_5min(ticker, &sixty_days_ago, &today).await {
|
}
|
||||||
if !prices.is_empty() {
|
|
||||||
save_prices_for_ticker(ticker, "5min", prices.clone()).await?;
|
|
||||||
println!(" Saved {} 5min bars for {ticker}", prices.len());
|
|
||||||
} else {
|
|
||||||
println!(" No 5min data available for {ticker} (market closed? retry later)");
|
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
println!(" 5min fetch failed for {ticker} (rate limit? try again)");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tokio::time::sleep(tokio::time::Duration::from_millis(250)).await;
|
// Ensure company directory exists (now uses LEI)
|
||||||
|
//let figi_dir = format!("data/companies_by_figi/{}/", company.primary_figi);
|
||||||
|
ensure_company_dirs(&company.lei).await?;
|
||||||
|
save_company_metadata(&company).await?;
|
||||||
|
|
||||||
|
// === STEP 1: Discover additional exchanges using each known ISIN ===
|
||||||
|
let mut all_tickers = company.tickers.clone();
|
||||||
|
|
||||||
|
if let Some(primary_ticker) = company.tickers.iter().find(|t| t.primary) {
|
||||||
|
println!(" Discovering additional exchanges across {} ISIN(s)...", company.isins.len());
|
||||||
|
|
||||||
|
for isin in &company.isins {
|
||||||
|
println!(" → Checking ISIN: {}", isin);
|
||||||
|
match discover_available_exchanges(isin, &primary_ticker.ticker).await {
|
||||||
|
Ok(discovered) => {
|
||||||
|
if discovered.is_empty() {
|
||||||
|
println!(" – No new exchanges found for {}", isin);
|
||||||
|
} else {
|
||||||
|
for disc in discovered {
|
||||||
|
if !all_tickers.iter().any(|t| t.ticker == disc.ticker && t.exchange_mic == disc.exchange_mic) {
|
||||||
|
println!(" New equity listing → {} ({}) via ISIN {}",
|
||||||
|
disc.ticker, disc.exchange_mic, isin);
|
||||||
|
all_tickers.push(disc);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => println!(" Discovery failed for {}: {}", isin, e),
|
||||||
|
}
|
||||||
|
tokio::time::sleep(tokio::time::Duration::from_millis(300)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save updated metadata if we found new listings
|
||||||
|
if all_tickers.len() > company.tickers.len() {
|
||||||
|
company.tickers = all_tickers.clone();
|
||||||
|
save_company_metadata(&company).await?;
|
||||||
|
println!(" Updated metadata: {} total tickers", all_tickers.len());
|
||||||
|
}
|
||||||
|
|
||||||
|
// === STEP 2: Fetch data from ALL available tickers ===
|
||||||
|
for ticker_info in &all_tickers {
|
||||||
|
let ticker = &ticker_info.ticker;
|
||||||
|
println!(" → Fetching: {} ({})", ticker, ticker_info.exchange_mic);
|
||||||
|
|
||||||
|
let mut daily_success = false;
|
||||||
|
let mut intraday_success = false;
|
||||||
|
|
||||||
|
// Earnings: only fetch from primary ticker to avoid duplicates
|
||||||
|
if ticker_info.primary {
|
||||||
|
if let Ok(new_events) = fetch_earnings_history(client, ticker).await {
|
||||||
|
let result = process_batch(&new_events, &mut existing_events, &today);
|
||||||
|
save_changes(&result.changes).await?;
|
||||||
|
println!(" Earnings events: {}", new_events.len());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Daily prices
|
||||||
|
if let Ok(prices) = fetch_daily_price_history(ticker, &config.corporate_start_date, &today).await {
|
||||||
|
if !prices.is_empty() {
|
||||||
|
save_prices_by_source(&company.lei, ticker, "daily", prices).await?;
|
||||||
|
daily_success = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5-minute intraday (last 60 days)
|
||||||
|
let sixty_days_ago = (chrono::Local::now() - chrono::Duration::days(60))
|
||||||
|
.format("%Y-%m-%d")
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
if let Ok(prices) = fetch_price_history_5min(ticker, &sixty_days_ago, &today).await {
|
||||||
|
if !prices.is_empty() {
|
||||||
|
save_prices_by_source(&company.lei, ticker, "5min", prices).await?;
|
||||||
|
intraday_success = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update available_exchanges.json (now under LEI folder)
|
||||||
|
update_available_exchange(
|
||||||
|
&company.lei,
|
||||||
|
ticker,
|
||||||
|
&ticker_info.exchange_mic,
|
||||||
|
daily_success,
|
||||||
|
intraday_success,
|
||||||
|
).await?;
|
||||||
|
|
||||||
|
tokio::time::sleep(tokio::time::Duration::from_millis(800)).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
// === STEP 3: Aggregate all sources into unified USD prices ===
|
||||||
|
println!(" Aggregating multi-source price data (FX-adjusted)...");
|
||||||
|
if let Err(e) = aggregate_best_price_data(&company.lei).await {
|
||||||
|
println!(" Aggregation failed: {}", e);
|
||||||
|
} else {
|
||||||
|
println!(" Aggregation complete");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
save_optimized_events(existing).await?;
|
// Final save of optimized earnings events
|
||||||
|
save_optimized_events(existing_events).await?;
|
||||||
|
println!("\nCorporate update complete (LEI-based)");
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn enrich_companies_with_leis(
|
||||||
|
companies: &mut Vec<CompanyMetadata>,
|
||||||
|
lei_to_isins: &HashMap<String, Vec<String>>,
|
||||||
|
) {
|
||||||
|
for company in companies.iter_mut() {
|
||||||
|
if company.lei.is_empty() {
|
||||||
|
// Try to find LEI by any known ISIN
|
||||||
|
for isin in &company.isins {
|
||||||
|
for (lei, isins) in lei_to_isins {
|
||||||
|
if isins.contains(isin) {
|
||||||
|
company.lei = lei.clone();
|
||||||
|
println!("Found real LEI {} for {}", lei, company.name);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !company.lei.is_empty() { break; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: generate fake LEI if still missing
|
||||||
|
if company.lei.is_empty() {
|
||||||
|
company.lei = format!("FAKE{:019}", rand::random::<u64>());
|
||||||
|
println!("No real LEI found → using fake for {}", company.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub struct ProcessResult {
|
pub struct ProcessResult {
|
||||||
pub changes: Vec<CompanyEventChange>,
|
pub changes: Vec<CompanyEventChange>,
|
||||||
}
|
}
|
||||||
|
|||||||
58
src/data/companies.json
Normal file
58
src/data/companies.json
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"lei": "8I5D5ASD7N5Z5P2K9M3J",
|
||||||
|
"isins": ["US46625H1005"],
|
||||||
|
"primary_isin": "US46625H1005",
|
||||||
|
"name": "JPMorgan Chase & Co.",
|
||||||
|
"tickers": [
|
||||||
|
{ "ticker": "JPM", "exchange_mic": "XNYS", "currency": "USD", "primary": true },
|
||||||
|
{ "ticker": "JPM-PC", "exchange_mic": "XNYS", "currency": "USD", "primary": false }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"lei": "5493001KJTIIGC8Y1R12",
|
||||||
|
"isins": ["US5949181045"],
|
||||||
|
"primary_isin": "US5949181045",
|
||||||
|
"name": "Microsoft Corporation",
|
||||||
|
"tickers": [
|
||||||
|
{ "ticker": "MSFT", "exchange_mic": "XNAS", "currency": "USD", "primary": true }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"lei": "529900T8BM49AURSDO55",
|
||||||
|
"isins": ["CNE000001P37"],
|
||||||
|
"primary_isin": "CNE000001P37",
|
||||||
|
"name": "Industrial and Commercial Bank of China",
|
||||||
|
"tickers": [
|
||||||
|
{ "ticker": "601398.SS", "exchange_mic": "XSHG", "currency": "CNY", "primary": true },
|
||||||
|
{ "ticker": "1398.HK", "exchange_mic": "XHKG", "currency": "HKD", "primary": false }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"lei": "519900X5W8K6C1FZ3B57",
|
||||||
|
"isins": ["JP3702200000"],
|
||||||
|
"primary_isin": "JP3702200000",
|
||||||
|
"name": "Toyota Motor Corporation",
|
||||||
|
"tickers": [
|
||||||
|
{ "ticker": "7203.T", "exchange_mic": "XJPX", "currency": "JPY", "primary": true },
|
||||||
|
{ "ticker": "TM", "exchange_mic": "XNYS", "currency": "USD", "primary": false }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"lei": "529900T8BM49AURSDO56",
|
||||||
|
"isins": ["HK0000069689"],
|
||||||
|
"primary_isin": "HK0000069689",
|
||||||
|
"name": "Tencent Holdings Limited",
|
||||||
|
"tickers": [
|
||||||
|
{ "ticker": "0700.HK", "exchange_mic": "XHKG", "currency": "HKD", "primary": true },
|
||||||
|
{ "ticker": "TCEHY", "exchange_mic": "OTCM", "currency": "USD", "primary": false }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"lei": "8I5D5Q1L7N5Z5P2K9M3J",
|
||||||
|
"isins": ["US90953F1049"],
|
||||||
|
"primary_isin": "US90953F1049",
|
||||||
|
"name": "Test Bonds Filter",
|
||||||
|
"tickers": [{ "ticker": "JPM", "exchange_mic": "XNYS", "currency": "USD", "primary": true }]
|
||||||
|
}
|
||||||
|
]
|
||||||
6
src/data/index.txt
Normal file
6
src/data/index.txt
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
data/*
|
||||||
|
|
||||||
|
companies.json
|
||||||
|
continents.json
|
||||||
|
countries.json
|
||||||
|
exchanges.json
|
||||||
32
src/main.rs
32
src/main.rs
@@ -4,7 +4,8 @@ mod corporate;
|
|||||||
mod config;
|
mod config;
|
||||||
mod util;
|
mod util;
|
||||||
|
|
||||||
use fantoccini::{ClientBuilder, Locator};
|
use fantoccini::{ClientBuilder};
|
||||||
|
use serde_json::{Map, Value};
|
||||||
use tokio::signal;
|
use tokio::signal;
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
@@ -17,11 +18,31 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
|
|
||||||
// === Start ChromeDriver ===
|
// === Start ChromeDriver ===
|
||||||
let mut child = std::process::Command::new("chromedriver-win64/chromedriver.exe")
|
let mut child = std::process::Command::new("chromedriver-win64/chromedriver.exe")
|
||||||
.args(["--port=9515"])
|
.args(["--port=9515"]) // Level 3 = minimal logs
|
||||||
.spawn()?;
|
.spawn()?;
|
||||||
|
|
||||||
let client = ClientBuilder::native()
|
// Build capabilities to hide infobar + enable full rendering
|
||||||
.connect("http://localhost:9515")
|
let port = 9515;
|
||||||
|
let caps_value = serde_json::json!({
|
||||||
|
"goog:chromeOptions": {
|
||||||
|
"args": [
|
||||||
|
//"--headless",
|
||||||
|
"--disable-gpu",
|
||||||
|
"--disable-notifications",
|
||||||
|
"--disable-popup-blocking",
|
||||||
|
"--disable-blink-features=AutomationControlled"
|
||||||
|
],
|
||||||
|
"excludeSwitches": ["enable-automation"]
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let caps_map: Map<String, Value> = caps_value.as_object()
|
||||||
|
.expect("Capabilities should be a JSON object")
|
||||||
|
.clone();
|
||||||
|
|
||||||
|
let mut client = ClientBuilder::native()
|
||||||
|
.capabilities(caps_map)
|
||||||
|
.connect(&format!("http://localhost:{}", port))
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Graceful shutdown
|
// Graceful shutdown
|
||||||
@@ -39,8 +60,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
|
|
||||||
// === Corporate Earnings Update ===
|
// === Corporate Earnings Update ===
|
||||||
println!("\nUpdating Corporate Earnings");
|
println!("\nUpdating Corporate Earnings");
|
||||||
let tickers = config::get_tickers();
|
corporate::run_full_update(&client, &config).await?;
|
||||||
corporate::run_full_update(&client, tickers, &config).await?;
|
|
||||||
|
|
||||||
// === Cleanup ===
|
// === Cleanup ===
|
||||||
client.close().await?;
|
client.close().await?;
|
||||||
|
|||||||
@@ -9,15 +9,14 @@ pub async fn ensure_data_dirs() -> anyhow::Result<()> {
|
|||||||
"economic_event_changes",
|
"economic_event_changes",
|
||||||
"corporate_events",
|
"corporate_events",
|
||||||
"corporate_prices",
|
"corporate_prices",
|
||||||
|
"data",
|
||||||
];
|
];
|
||||||
|
|
||||||
for dir in dirs {
|
for dir in dirs {
|
||||||
let path = Path::new(dir);
|
let path = Path::new(dir);
|
||||||
if !path.exists() {
|
if !path.exists() {
|
||||||
fs::create_dir_all(path).await?;
|
tokio::fs::create_dir_all(path).await?;
|
||||||
println!("Created directory: {dir}");
|
println!("Created directory: {dir}");
|
||||||
}
|
}
|
||||||
// else → silently continue
|
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
Reference in New Issue
Block a user