Optimize scraping logic by skipping verified NamespaceRequests
- Added logic to skip scraping if a NamespaceRequest is already verified and the last scrape was within the past day. - This change reduces unnecessary scraping, improving efficiency and reducing load on target servers. - Added logging to indicate when a scrape is skipped due to verification status. modified: remarkbox/models/namespace_request.py
This commit is contained in:
parent
c5453d632d
commit
41ab711fe3
1 changed files with 8 additions and 3 deletions
|
|
@ -68,11 +68,16 @@ class NamespaceRequest(RBase, Base):
|
|||
Return True if found else False.
|
||||
"""
|
||||
current_time = now_timestamp()
|
||||
one_day_in_milliseconds = 24 * 60 * 60 * 1000
|
||||
|
||||
# Skip scraping if already verified and last scrape was within a day
|
||||
if (
|
||||
self.last_scrape_timestamp
|
||||
and (current_time - self.last_scrape_timestamp) < 300
|
||||
self.verified
|
||||
and self.last_scrape_timestamp
|
||||
and (current_time - self.last_scrape_timestamp) < one_day_in_milliseconds
|
||||
):
|
||||
return self.verified
|
||||
log.info(f"Skipping scrape for verified namespace request id={self.id}")
|
||||
return True
|
||||
|
||||
# Extract domain from target using miniuri
|
||||
domain = self._get_domain_from_target(self.target)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue