From c5453d632d39883b0633541d5c876bb0b135acac Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Sun, 6 Oct 2024 16:54:10 -0400 Subject: [PATCH] Enhance scrape_target method to handle HTTP redirects - Implemented logic to follow HTTP redirects up to a specified maximum depth (max_redirects) in the scrape_target method. - Added logging to track each redirect step, providing better visibility into the redirect chain. - Set a limit on the number of redirects to prevent infinite loops in case of circular redirects. - This change ensures that the UUID is checked on the final destination page, improving the accuracy of the scraping process. modified: remarkbox/models/namespace_request.py --- remarkbox/models/namespace_request.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/remarkbox/models/namespace_request.py b/remarkbox/models/namespace_request.py index f4373d8..b26a74a 100644 --- a/remarkbox/models/namespace_request.py +++ b/remarkbox/models/namespace_request.py @@ -12,6 +12,7 @@ from miniuri import Uri log = logging.getLogger(__name__) +# Dictionary to track when scraping is allowed again for specific domains domain_skip_until = {} @@ -62,7 +63,7 @@ class NamespaceRequest(RBase, Base): else: self.unverify() - def scrape_target(self): + def scrape_target(self, max_redirects=5): """Scrape the given target for NamespaceRequest id Return True if found else False. """ @@ -106,7 +107,19 @@ class NamespaceRequest(RBase, Base): ) ) try: - resp = requests.get(self.target, timeout=8.50) + # Follow redirects up to max_redirects + resp = requests.get( + self.target, headers=headers, timeout=8.50, allow_redirects=True + ) + redirect_count = 0 + while resp.is_redirect and redirect_count < max_redirects: + redirect_count += 1 + next_url = resp.headers.get("Location") + if not next_url: + break + log.info(f"Redirecting to {next_url}") + resp = requests.get(next_url, headers=headers, timeout=8.50) + if resp.ok: if namespace_request_id in resp.text: log.info(