From 41ab711fe3f421e5159a6ed29669544ad44c56cf Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Sun, 6 Oct 2024 18:41:49 -0400 Subject: [PATCH] Optimize scraping logic by skipping verified NamespaceRequests - Added logic to skip scraping if a NamespaceRequest is already verified and the last scrape was within the past day. - This change reduces unnecessary scraping, improving efficiency and reducing load on target servers. - Added logging to indicate when a scrape is skipped due to verification status. modified: remarkbox/models/namespace_request.py --- remarkbox/models/namespace_request.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/remarkbox/models/namespace_request.py b/remarkbox/models/namespace_request.py index b26a74a..49df562 100644 --- a/remarkbox/models/namespace_request.py +++ b/remarkbox/models/namespace_request.py @@ -68,11 +68,16 @@ class NamespaceRequest(RBase, Base): Return True if found else False. """ current_time = now_timestamp() + one_day_in_milliseconds = 24 * 60 * 60 * 1000 + + # Skip scraping if already verified and last scrape was within a day if ( - self.last_scrape_timestamp - and (current_time - self.last_scrape_timestamp) < 300 + self.verified + and self.last_scrape_timestamp + and (current_time - self.last_scrape_timestamp) < one_day_in_milliseconds ): - return self.verified + log.info(f"Skipping scrape for verified namespace request id={self.id}") + return True # Extract domain from target using miniuri domain = self._get_domain_from_target(self.target)