From ffdbdaad68366a1467f7da8aaf9e7652023e7748 Mon Sep 17 00:00:00 2001
From: Russell Ballestrini <russell@unturf.com>
Date: Thu, 1 Jan 2026 15:49:04 -0500
Subject: [PATCH] Add hydra state persistence for delta URL detection

- Hydra state file (hydra-{domain}.json) never rotates
- Tracks seen URLs with first_seen timestamp, title, published date
- _filter_new_hydra_urls() returns only NEW URLs not seen before
- Enables detecting new posts when sites publish via RSS/Atom/Sitemap
---
 neopig.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 64 insertions(+), 2 deletions(-)

diff --git a/neopig.py b/neopig.py
index c50b685..147087e 100644
--- a/neopig.py
+++ b/neopig.py
@@ -416,6 +416,64 @@ class NeoPig:
         except Exception as e:
             logger.debug(f"Failed to save state: {e}")
 
+    def _get_hydra_state_file(self, domain: str) -> Path:
+        """Get path to hydra state file for a domain (never rotated)."""
+        safe_domain = domain.replace('/', '_').replace(':', '_')
+        return self._state_dir / f"hydra-{safe_domain}.json"
+
+    def _load_hydra_state(self, domain: str) -> Dict[str, Any]:
+        """Load hydra state - URLs seen from feeds (never cleared)."""
+        state_file = self._get_hydra_state_file(domain)
+        if not state_file.exists():
+            return {'seen_urls': {}, 'feeds': {}}
+        try:
+            return json.loads(state_file.read_text())
+        except Exception as e:
+            logger.warning(f"Failed to load hydra state: {e}")
+            return {'seen_urls': {}, 'feeds': {}}
+
+    def _save_hydra_state(self, domain: str, state: Dict[str, Any]):
+        """Save hydra state (persists across crawls, never rotated)."""
+        state_file = self._get_hydra_state_file(domain)
+        try:
+            self._state_dir.mkdir(parents=True, exist_ok=True)
+            state['last_updated'] = datetime.now(timezone.utc).isoformat()
+            with open(state_file, 'w') as f:
+                json.dump(state, f, indent=2)
+        except Exception as e:
+            logger.warning(f"Failed to save hydra state: {e}")
+
+    def _filter_new_hydra_urls(self, domain: str, feed_items: List) -> List:
+        """Filter feed items to only return NEW URLs not seen before.
+
+        Also updates hydra state with newly seen URLs.
+        Returns list of new FeedItem objects.
+        """
+        state = self._load_hydra_state(domain)
+        seen_urls = state.get('seen_urls', {})
+
+        new_items = []
+        now = datetime.now(timezone.utc).isoformat()
+
+        for item in feed_items:
+            url = item.url
+            if url not in seen_urls:
+                new_items.append(item)
+                seen_urls[url] = {
+                    'first_seen': now,
+                    'title': item.title,
+                    'published': item.published,
+                }
+
+        if new_items:
+            state['seen_urls'] = seen_urls
+            self._save_hydra_state(domain, state)
+            logger.info(f"Hydra: {len(new_items)} NEW URLs (of {len(feed_items)} total)")
+        else:
+            logger.info(f"Hydra: No new URLs found (all {len(feed_items)} already seen)")
+
+        return new_items
+
     def _load_state(self, target_url: str) -> bool:
         """Load saved crawl state. Returns True if state was loaded.
 
@@ -897,8 +955,12 @@ class NeoPig:
             logger.info("=== HYDRA MODE: Fetching feeds/sitemaps ===")
             feed_items = await self.fetcher.fetch_feeds(target_uri)
             if feed_items:
-                hydra_urls = [item.url for item in feed_items]
-                logger.info(f"Hydra mode: Found {len(hydra_urls)} URLs from feeds/sitemaps")
+                # Filter to only NEW URLs not seen before (persists across crawls)
+                domain = Uri(target_uri).host
+                new_items = self._filter_new_hydra_urls(domain, feed_items)
+                if new_items:
+                    hydra_urls = [item.url for item in new_items]
+                    logger.info(f"Hydra mode: Injecting {len(hydra_urls)} NEW URLs into crawl queue")
 
         logger.info("=== STREAMING ETL: HTML + Media + Screenshots ===")
         pages = await self.fetcher.fetch_with_depth(