Optimize scraping logic by skipping verified NamespaceRequests

- Added logic to skip scraping if a NamespaceRequest is already verified and the last scrape was within the past day.
- This change reduces unnecessary scraping, improving efficiency and reducing load on target servers.
- Added logging to indicate when a scrape is skipped due to verification status.

	modified:   remarkbox/models/namespace_request.py
This commit is contained in:
Russell Ballestrini 2024-10-06 18:41:49 -04:00
parent c5453d632d
commit 41ab711fe3

View file

@ -68,11 +68,16 @@ class NamespaceRequest(RBase, Base):
Return True if found else False.
"""
current_time = now_timestamp()
one_day_in_milliseconds = 24 * 60 * 60 * 1000
# Skip scraping if already verified and last scrape was within a day
if (
self.last_scrape_timestamp
and (current_time - self.last_scrape_timestamp) < 300
self.verified
and self.last_scrape_timestamp
and (current_time - self.last_scrape_timestamp) < one_day_in_milliseconds
):
return self.verified
log.info(f"Skipping scrape for verified namespace request id={self.id}")
return True
# Extract domain from target using miniuri
domain = self._get_domain_from_target(self.target)