added vpn ip rotation
This commit is contained in:
397
PRACTICAL_EXAMPLES.md
Normal file
397
PRACTICAL_EXAMPLES.md
Normal file
@@ -0,0 +1,397 @@
|
||||
// PRACTICAL EXAMPLES: Integration in Economic & Corporate Module
|
||||
// ================================================================
|
||||
// Diese Datei zeigt konkrete Implementierungen für die VPN-Integration
|
||||
// in die bestehenden economic:: und corporate:: Module
|
||||
|
||||
use anyhow::Result;
|
||||
use std::sync::Arc;
|
||||
use tokio::time::{sleep, Duration};
|
||||
|
||||
// ============================================================================
|
||||
// EXAMPLE 1: Vereinfachte Integration in economic::run_full_update()
|
||||
// ============================================================================
|
||||
|
||||
/// Beispiel: Economic Update mit VPN-Session-Management
|
||||
/// Kopieren Sie diese Struktur in src/economic/mod.rs
|
||||
///
|
||||
/// VORHER (ohne VPN):
|
||||
/// ```ignore
|
||||
/// pub async fn run_full_update(
|
||||
/// config: &Config,
|
||||
/// pool: &Arc<ChromeDriverPool>,
|
||||
/// ) -> Result<()> {
|
||||
/// let tickers = fetch_tickers().await?;
|
||||
/// for ticker in tickers {
|
||||
/// pool.execute(ticker, |client| async { /* scrape */ }).await?;
|
||||
/// }
|
||||
/// Ok(())
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// NACHHER (mit VPN):
|
||||
pub async fn example_economic_with_vpn(
|
||||
config: &crate::config::Config,
|
||||
pool: &Arc<crate::scraper::webdriver::ChromeDriverPool>,
|
||||
vpn: &crate::scraper::vpn_integration::VpnIntegration,
|
||||
) -> Result<()> {
|
||||
use crate::scraper::vpn_integration::VpnIntegration;
|
||||
|
||||
println!("📊 Running economic update with VPN support");
|
||||
|
||||
// Schritt 1: VPN initialisieren (falls aktiviert)
|
||||
if vpn.enabled {
|
||||
vpn.initialize_session().await?;
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
|
||||
// Schritt 2: Tickers/Events laden
|
||||
// let tickers = fetch_economic_events().await?;
|
||||
let tickers = vec!["example1", "example2", "example3"]; // Mock
|
||||
|
||||
// Schritt 3: Für jeden Task
|
||||
for (idx, ticker) in tickers.iter().enumerate() {
|
||||
// A. Prüfe ob VPN-Rotation erforderlich
|
||||
if vpn.check_and_rotate_if_needed().await? {
|
||||
println!("🔄 Rotating VPN session...");
|
||||
sleep(Duration::from_secs(3)).await; // Warte auf neue IP
|
||||
}
|
||||
|
||||
// B. Führe Task aus
|
||||
match execute_economic_task(pool, ticker).await {
|
||||
Ok(_) => {
|
||||
// C. Inkrementiere Task-Counter
|
||||
vpn.increment_task().await;
|
||||
|
||||
// D. Logging
|
||||
if let Some(session_id) = vpn.get_current_session_id().await {
|
||||
println!(
|
||||
"✓ Task {}/{} completed in session {}",
|
||||
idx + 1,
|
||||
tickers.len(),
|
||||
session_id
|
||||
);
|
||||
} else {
|
||||
println!("✓ Task {}/{} completed", idx + 1, tickers.len());
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("❌ Task failed: {}", e);
|
||||
// Optional: Bei kritischen Fehlern brechen, sonst fortfahren
|
||||
}
|
||||
}
|
||||
|
||||
// E. Rate-Limiting (wichtig für Zielwebsite)
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
|
||||
println!("✓ Economic update completed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn execute_economic_task(
|
||||
_pool: &Arc<crate::scraper::webdriver::ChromeDriverPool>,
|
||||
_ticker: &str,
|
||||
) -> Result<()> {
|
||||
// TODO: Implementierung mit pool.execute()
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// EXAMPLE 2: Corporate Update mit VPN
|
||||
// ============================================================================
|
||||
|
||||
pub async fn example_corporate_with_vpn(
|
||||
config: &crate::config::Config,
|
||||
pool: &Arc<crate::scraper::webdriver::ChromeDriverPool>,
|
||||
vpn: &crate::scraper::vpn_integration::VpnIntegration,
|
||||
) -> Result<()> {
|
||||
println!("📊 Running corporate update with VPN support");
|
||||
|
||||
if vpn.enabled {
|
||||
vpn.initialize_session().await?;
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
|
||||
// Corporate tasks verarbeiten
|
||||
let companies = vec!["AAPL", "MSFT", "GOOGL"]; // Mock
|
||||
|
||||
for (idx, company) in companies.iter().enumerate() {
|
||||
// Rotation check
|
||||
if vpn.check_and_rotate_if_needed().await? {
|
||||
println!("🔄 Rotating VPN for corporate update");
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
}
|
||||
|
||||
// Task execution
|
||||
match execute_corporate_task(pool, company).await {
|
||||
Ok(_) => {
|
||||
vpn.increment_task().await;
|
||||
println!("✓ Corporate task {}/{} completed", idx + 1, companies.len());
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("❌ Corporate task failed: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
|
||||
println!("✓ Corporate update completed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn execute_corporate_task(
|
||||
_pool: &Arc<crate::scraper::webdriver::ChromeDriverPool>,
|
||||
_company: &str,
|
||||
) -> Result<()> {
|
||||
// TODO: Implementierung
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// EXAMPLE 3: Advanced - Custom VPN-Rotation pro Task
|
||||
// ============================================================================
|
||||
|
||||
/// Wenn Sie eine IP pro Task haben möchten (nicht empfohlen, aber möglich):
|
||||
pub async fn example_rotation_per_task(
|
||||
pool: &Arc<crate::scraper::webdriver::ChromeDriverPool>,
|
||||
vpn: &crate::scraper::vpn_integration::VpnIntegration,
|
||||
) -> Result<()> {
|
||||
let tasks = vec!["task1", "task2", "task3"];
|
||||
|
||||
for task in tasks {
|
||||
// Vor jedem Task: Neue Session erstellen
|
||||
if vpn.enabled {
|
||||
vpn.initialize_session().await?;
|
||||
sleep(Duration::from_secs(5)).await; // Warte auf Verbindung
|
||||
|
||||
if let Some(ip) = vpn.get_current_ip().await {
|
||||
println!("📍 Task '{}' uses IP: {}", task, ip);
|
||||
}
|
||||
}
|
||||
|
||||
// Task ausführen
|
||||
println!("Executing task: {}", task);
|
||||
|
||||
// Nach Task: Task-Counter (hier nur 1)
|
||||
vpn.increment_task().await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// EXAMPLE 4: Error Handling & Retry Logic
|
||||
// ============================================================================
|
||||
|
||||
pub async fn example_with_retry(
|
||||
pool: &Arc<crate::scraper::webdriver::ChromeDriverPool>,
|
||||
vpn: &crate::scraper::vpn_integration::VpnIntegration,
|
||||
max_retries: u32,
|
||||
) -> Result<()> {
|
||||
let tasks = vec!["task1", "task2"];
|
||||
|
||||
for task in tasks {
|
||||
let mut attempt = 0;
|
||||
|
||||
loop {
|
||||
attempt += 1;
|
||||
|
||||
// Rotation check
|
||||
if vpn.check_and_rotate_if_needed().await? {
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
}
|
||||
|
||||
// Versuche Task
|
||||
match execute_economic_task(pool, task).await {
|
||||
Ok(_) => {
|
||||
vpn.increment_task().await;
|
||||
println!("✓ Task succeeded on attempt {}", attempt);
|
||||
break;
|
||||
}
|
||||
Err(e) if attempt < max_retries => {
|
||||
eprintln!("⚠️ Task failed (attempt {}): {}, retrying...", attempt, e);
|
||||
|
||||
// Exponential backoff
|
||||
let backoff = Duration::from_secs(2 ^ (attempt - 1));
|
||||
sleep(backoff).await;
|
||||
|
||||
// Optional: Neue VPN-Session vor Retry
|
||||
if attempt % 2 == 0 && vpn.enabled {
|
||||
println!("🔄 Rotating VPN before retry");
|
||||
vpn.initialize_session().await?;
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("❌ Task failed after {} attempts: {}", max_retries, e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// EXAMPLE 5: Batch Processing (mehrere Tasks pro Session)
|
||||
// ============================================================================
|
||||
|
||||
pub async fn example_batch_processing(
|
||||
pool: &Arc<crate::scraper::webdriver::ChromeDriverPool>,
|
||||
vpn: &crate::scraper::vpn_integration::VpnIntegration,
|
||||
batch_size: usize,
|
||||
) -> Result<()> {
|
||||
let all_tasks = vec!["t1", "t2", "t3", "t4", "t5"];
|
||||
|
||||
// Gruppiere Tasks in Batches
|
||||
for batch in all_tasks.chunks(batch_size) {
|
||||
// Neue Session pro Batch
|
||||
if vpn.enabled {
|
||||
vpn.initialize_session().await?;
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
|
||||
if let Some(ip) = vpn.get_current_ip().await {
|
||||
println!("🔗 New batch session with IP: {}", ip);
|
||||
}
|
||||
}
|
||||
|
||||
// Tasks in Batch verarbeiten
|
||||
for task in batch {
|
||||
if let Ok(_) = execute_economic_task(pool, task).await {
|
||||
vpn.increment_task().await;
|
||||
println!("✓ Task {} completed", task);
|
||||
}
|
||||
}
|
||||
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// EXAMPLE 6: Parallel Scraping mit VPN-Awareness
|
||||
// ============================================================================
|
||||
|
||||
/// Nutze ChromeDriver-Pool-Parallelism mit VPN
|
||||
pub async fn example_parallel_with_vpn(
|
||||
pool: &Arc<crate::scraper::webdriver::ChromeDriverPool>,
|
||||
vpn: &crate::scraper::vpn_integration::VpnIntegration,
|
||||
) -> Result<()> {
|
||||
let tasks = vec!["url1", "url2", "url3"];
|
||||
|
||||
// Stellt sicher, dass nur pool_size Tasks parallel laufen
|
||||
// (Semaphore im ChromeDriverPool kontrolliert das)
|
||||
let mut handles = vec![];
|
||||
|
||||
for task in tasks {
|
||||
let vpn_clone = std::sync::Arc::new(
|
||||
crate::scraper::vpn_integration::VpnIntegration::from_config(&crate::config::Config::default())?
|
||||
);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
// Jeder Task rotiert unabhängig
|
||||
vpn_clone.increment_task().await;
|
||||
println!("Task {} executed", task);
|
||||
});
|
||||
|
||||
handles.push(handle);
|
||||
}
|
||||
|
||||
// Warte auf alle Tasks
|
||||
for handle in handles {
|
||||
handle.await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// EXAMPLE 7: Monitoring & Stats
|
||||
// ============================================================================
|
||||
|
||||
pub struct VpnSessionStats {
|
||||
pub total_sessions: usize,
|
||||
pub total_tasks: usize,
|
||||
pub tasks_per_session: Vec<usize>,
|
||||
pub ips_used: Vec<String>,
|
||||
}
|
||||
|
||||
pub async fn collect_stats(
|
||||
vpn: &crate::scraper::vpn_integration::VpnIntegration,
|
||||
) -> VpnSessionStats {
|
||||
// TODO: Sammeln von Statistiken
|
||||
// In echtem Code würde man einen Analytics-Service haben
|
||||
|
||||
VpnSessionStats {
|
||||
total_sessions: 0,
|
||||
total_tasks: 0,
|
||||
tasks_per_session: vec![],
|
||||
ips_used: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn print_stats(stats: &VpnSessionStats) {
|
||||
println!("\n📊 VPN Session Statistics:");
|
||||
println!(" Total sessions: {}", stats.total_sessions);
|
||||
println!(" Total tasks: {}", stats.total_tasks);
|
||||
println!(" Avg tasks/session: {}",
|
||||
if stats.total_sessions > 0 {
|
||||
stats.total_tasks / stats.total_sessions
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
println!(" Unique IPs: {}", stats.ips_used.len());
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// EXAMPLE 8: Integration in main.rs
|
||||
// ============================================================================
|
||||
|
||||
/// Wie Sie alles in main.rs zusammenbringen:
|
||||
///
|
||||
/// ```ignore
|
||||
/// #[tokio::main]
|
||||
/// async fn main() -> Result<()> {
|
||||
/// // 1. Setup
|
||||
/// tracing_subscriber::fmt().init();
|
||||
/// let config = Config::load()?;
|
||||
///
|
||||
/// // 2. VPN initialisieren
|
||||
/// let vpn = VpnIntegration::from_config(&config)?;
|
||||
///
|
||||
/// // 3. Pool erstellen
|
||||
/// let pool = Arc::new(ChromeDriverPool::new(config.max_parallel_tasks).await?);
|
||||
///
|
||||
/// // 4. Updates mit VPN
|
||||
/// economic::run_full_update_with_vpn(&config, &pool, &vpn).await?;
|
||||
/// corporate::run_full_update_with_vpn(&config, &pool, &vpn).await?;
|
||||
///
|
||||
/// Ok(())
|
||||
/// }
|
||||
/// ```
|
||||
|
||||
// ============================================================================
|
||||
// EXAMPLE 9: Unit Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rotation_trigger() {
|
||||
// Mock VPN-Integration testen
|
||||
let vpn = crate::scraper::vpn_integration::VpnIntegration {
|
||||
session_manager: None,
|
||||
automater: None,
|
||||
enabled: false,
|
||||
};
|
||||
|
||||
assert!(!vpn.enabled);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user