diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a97faa..0247417 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- **Detailed crawl diagnostic logging**: Added comprehensive logging to diagnose crawl behavior + - Log crawl parameters at start (max_depth, initial URLs count, query keywords) + - Debug log showing crawl queue state during processing + - Detailed link extraction statistics (total found, added to queue, skipped by reason) + - Separate counters for links skipped due to: robots.txt, wrong domain, or max depth reached + - Applied to both fresh page fetching and cached page link extraction + - Located in `unturf_spider.py:611-613` (crawl_recursive start logging) + - Located in `unturf_spider.py:547` (cached page link extraction logging) + - Located in `unturf_spider.py:596` (fresh page link extraction logging) + ### Fixed - **Critical depth traversal bug**: Fixed issue where cached pages at depth 0 would not extract links, preventing depth 1+ crawling - When a page was already cached and `force_crawl=False`, the crawler would return early without extracting links diff --git a/unturf_spider.py b/unturf_spider.py index b17b077..826f23c 100644 --- a/unturf_spider.py +++ b/unturf_spider.py @@ -510,6 +510,9 @@ class BaseCrawler: # Extract links and return them for further crawling new_urls = [] + skipped_robots = 0 + skipped_domain = 0 + skipped_depth = 0 for link in links: parsed = urlparse(link) if parsed.scheme not in ("http", "https"): @@ -530,9 +533,18 @@ class BaseCrawler: trust_linked_domains and not is_base_or_subdomain and depth == 0 ): should_crawl = True + if should_crawl: new_urls.append((f"Linked from {url}", link)) - logger.info(f"Found {len(new_urls)} links to crawl at depth {depth+1}") + else: + if not is_base_or_subdomain: + skipped_domain += 1 + else: + skipped_depth += 1 + else: + skipped_robots += 1 + + logger.info(f"Link extraction (cached page): found {len(links)} total, added {len(new_urls)} to crawl queue, skipped {skipped_robots} (robots.txt), skipped {skipped_domain} (wrong domain), skipped {skipped_depth} (max depth reached)") return new_urls else: # At max depth, no need to extract links @@ -558,6 +570,9 @@ class BaseCrawler: page_linked_domains = set() new_urls = [] + skipped_robots = 0 + skipped_domain = 0 + skipped_depth = 0 for link in links: parsed = urlparse(link) if parsed.scheme not in ("http", "https"): @@ -579,8 +594,18 @@ class BaseCrawler: trust_linked_domains and not is_base_or_subdomain and depth == 0 ): should_crawl = True + if should_crawl: new_urls.append((f"Linked from {url}", link)) + else: + if not is_base_or_subdomain: + skipped_domain += 1 + else: + skipped_depth += 1 + else: + skipped_robots += 1 + + logger.info(f"Link extraction: found {len(links)} total, added {len(new_urls)} to crawl queue, skipped {skipped_robots} (robots.txt), skipped {skipped_domain} (wrong domain), skipped {skipped_depth} (max depth reached)") self.cache_article( url, page_title, @@ -608,7 +633,9 @@ class BaseCrawler: ): to_crawl = [(title_url, 0) for title_url in start_urls] all_urls = [] + logger.info(f"Starting crawl with max_depth={max_depth}, initial URLs={len(start_urls)}, query_keywords={query_keywords}") while to_crawl: + logger.debug(f"Crawl queue: {len(to_crawl)} URLs remaining, {len(all_urls)} URLs processed so far") current_title_url, depth = to_crawl.pop(0) new_urls = self.process_url( current_title_url,