engineering/pig.py
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve depth CLI help and add max depth validation

Browse source 
- Clarify depth help: 0=single page, 1=page+links, -1=unlimited
- Reject depth >18, suggest using -1 for unlimited crawling
- Update CLAUDE.md with depth documentation
This commit is contained in:
Russell Ballestrini 2025-12-31 15:37:18 -05:00
parent 8d669e206c
commit b0542fccc6
3 changed files with 13 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
CLAUDE.md
View file

@ -69,7 +69,7 @@ stats = await pig.crawl(
target_uri="https://example.com",
keywords=["tag1", "tag2"],
mode=CrawlMode.IMAGES,
depth=-1, # unlimited
depth=-1, # 0=single page, 1=page+links, -1=unlimited
max_pages=-1,
)
```

7
archive.py
View file

@ -668,7 +668,7 @@ async def main():
parser.add_argument("url", nargs='?', help="URL of the site to archive")
parser.add_argument("-o", "--output", default=".", help="Output directory for tar.gz")
parser.add_argument("-d", "--depth", type=int, default=-1, help="Crawl depth (-1 = unlimited)")
parser.add_argument("-d", "--depth", type=int, default=-1, help="Crawl depth: 0=single page, 1=page+links, -1=unlimited")
parser.add_argument("-p", "--max-pages", type=int, default=-1, help="Max pages (-1 = unlimited)")
parser.add_argument("--no-screenshot", action="store_true", help="Disable screenshots")
parser.add_argument("--screenshot-width", type=int, default=1280, help="Screenshot width")
@ -687,6 +687,11 @@ async def main():
args = parser.parse_args()
# Validate depth - max 18 before suggesting unlimited
if args.depth > 18:
print(f"Error: depth {args.depth} is too high. Use --depth -1 for unlimited crawling.")
return
setup_logging(level=logging.DEBUG if args.verbose else logging.INFO)
# Handle --upgrade-neopig

7
neopig.py
View file

@ -1706,7 +1706,7 @@ async def main():
"-d", "--depth",
type=int,
default=-1,
help="Crawl depth (-1 = unlimited, default: -1)"
help="Crawl depth: 0=single page, 1=page+links, -1=unlimited (default: -1)"
)
parser.add_argument(
@ -1856,6 +1856,11 @@ async def main():
args = parser.parse_args()
# Validate depth - max 18 before suggesting unlimited
if args.depth > 18:
print(f"Error: depth {args.depth} is too high. Use --depth -1 for unlimited crawling.")
return
# Create data directory if needed
from pathlib import Path
db_dir = Path(args.db).parent