diff --git a/Makefile b/Makefile index 09f90a7..0f5b64a 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ install: venv $(PIP) install -r requirements.txt test: install - $(PYTHON) test_crawl.py + $(PYTHON) -m pytest tests/ -v --tb=short crawl: install $(PYTHON) neopig.py $(ARGS) @@ -21,8 +21,8 @@ archive: install $(PYTHON) archive.py $(ARGS) # Build bootstrap binary for self-extracting archives -bootstrap: bootstrap.c - gcc -O2 -Wall -o bootstrap bootstrap.c -lz +bootstrap: scripts/bootstrap.c + gcc -O2 -Wall -o bootstrap scripts/bootstrap.c -lz @echo "Built: bootstrap ($$(stat -c%s bootstrap 2>/dev/null || stat -f%z bootstrap) bytes)" # Create self-extracting .run from a tarball diff --git a/archive.py b/archive.py index dfc6e3a..21deb91 100644 --- a/archive.py +++ b/archive.py @@ -54,9 +54,9 @@ from bs4 import BeautifulSoup from tqdm import tqdm from neopig import NeoPig, setup_logging, rotate_state_file, get_state_file_path -from async_web_fetcher import CrawlMode -from screenshot import ScreenshotConfig -from filevault import hash_to_path +from neopig.async_web_fetcher import CrawlMode +from neopig.screenshot import ScreenshotConfig +from neopig.filevault import hash_to_path # Optional markdown conversion try: diff --git a/rules_benedict.py b/data/rules_benedict.py similarity index 100% rename from rules_benedict.py rename to data/rules_benedict.py diff --git a/translations_body.py b/data/translations_body.py similarity index 100% rename from translations_body.py rename to data/translations_body.py diff --git a/neopig.py b/neopig.py index 5b4f363..e8568dd 100644 --- a/neopig.py +++ b/neopig.py @@ -38,7 +38,7 @@ from typing import List, Dict, Any, Optional, Set, Tuple from miniuri import Uri -from async_web_fetcher import ( +from neopig.async_web_fetcher import ( AsyncWebFetcher, CrawlMode, MediaItem, @@ -46,11 +46,11 @@ from async_web_fetcher import ( get_media_type_from_extension, get_media_type_from_mime, ) -from filevault import AsyncVault, hash_to_path -from database import Database, SCORE_SCREENSHOT, SCORE_OG_IMAGE, SCORE_THUMBNAIL, SCORE_FULL_RES -from screenshot import ScreenshotCapture, ScreenshotConfig -from domain_vault import VaultManager, DomainHtmlVault, DomainMediaVault, DomainLinkpeekVault, extract_media_urls -from repo import detect_vcs, clone_repo_async, pull_repo_async, get_repo_path, walk_files, get_commit_hash, get_file_language, is_binary_file +from neopig.filevault import AsyncVault, hash_to_path +from neopig.database import Database, SCORE_SCREENSHOT, SCORE_OG_IMAGE, SCORE_THUMBNAIL, SCORE_FULL_RES +from neopig.screenshot import ScreenshotCapture, ScreenshotConfig +from neopig.domain_vault import VaultManager, DomainHtmlVault, DomainMediaVault, DomainLinkpeekVault, extract_media_urls +from neopig.repo import detect_vcs, clone_repo_async, pull_repo_async, get_repo_path, walk_files, get_commit_hash, get_file_language, is_binary_file from tqdm import tqdm logger = logging.getLogger(__name__) diff --git a/async_filevault.py b/neopig/async_filevault.py similarity index 99% rename from async_filevault.py rename to neopig/async_filevault.py index 2f4acf5..b0d051b 100644 --- a/async_filevault.py +++ b/neopig/async_filevault.py @@ -19,7 +19,7 @@ import asyncio from functools import partial from typing import Any, Optional, List -from filevault import Vault, ensure_bytes, create_vault +from .filevault import Vault, ensure_bytes, create_vault __version__ = "1.1.0" __author__ = "Russell Ballestrini" diff --git a/async_web_fetcher.py b/neopig/async_web_fetcher.py similarity index 100% rename from async_web_fetcher.py rename to neopig/async_web_fetcher.py diff --git a/database.py b/neopig/database.py similarity index 100% rename from database.py rename to neopig/database.py diff --git a/domain_vault.py b/neopig/domain_vault.py similarity index 100% rename from domain_vault.py rename to neopig/domain_vault.py diff --git a/filevault.py b/neopig/filevault.py similarity index 100% rename from filevault.py rename to neopig/filevault.py diff --git a/html2md.py b/neopig/html2md.py similarity index 100% rename from html2md.py rename to neopig/html2md.py diff --git a/repo.py b/neopig/repo.py similarity index 100% rename from repo.py rename to neopig/repo.py diff --git a/screenshot.py b/neopig/screenshot.py similarity index 100% rename from screenshot.py rename to neopig/screenshot.py diff --git a/bootstrap-neopig.sh b/scripts/bootstrap-neopig.sh similarity index 100% rename from bootstrap-neopig.sh rename to scripts/bootstrap-neopig.sh diff --git a/bootstrap.c b/scripts/bootstrap.c similarity index 100% rename from bootstrap.c rename to scripts/bootstrap.c diff --git a/inject_rules.py b/scripts/inject_rules.py similarity index 100% rename from inject_rules.py rename to scripts/inject_rules.py diff --git a/inject_translations.py b/scripts/inject_translations.py similarity index 100% rename from inject_translations.py rename to scripts/inject_translations.py diff --git a/make-executable.sh b/scripts/make-executable.sh similarity index 100% rename from make-executable.sh rename to scripts/make-executable.sh diff --git a/serp/server.py b/serp/server.py index 9aac7fb..9c53f6a 100644 --- a/serp/server.py +++ b/serp/server.py @@ -30,8 +30,8 @@ from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, Redirect from sqlalchemy import text import uvicorn -from database import Database, OVER_9000 -from filevault import hash_to_path +from neopig.database import Database, OVER_9000 +from neopig.filevault import hash_to_path from miniuri import Uri from neopig.live import get_live_queue @@ -903,7 +903,7 @@ async def get_stats_endpoint(): # Add page count (handled separately since table may not exist) try: from sqlalchemy import select, func - from database import Page + from neopig.database import Page async with db.session() as session: result = await session.execute(select(func.count()).select_from(Page)) stats['total_pages'] = result.scalar() or 0 @@ -916,7 +916,7 @@ async def get_stats_endpoint(): async def random_item(type: str = Query(None, description="Type: media or page (random if not specified)")): """Redirect to a random media item or page.""" from sqlalchemy import select, func - from database import Media, Page + from neopig.database import Media, Page import random # If no type specified, randomly pick between media and page diff --git a/test_crawl.py b/tests/test_crawl.py similarity index 98% rename from test_crawl.py rename to tests/test_crawl.py index edfc123..c0bb874 100644 --- a/test_crawl.py +++ b/tests/test_crawl.py @@ -13,7 +13,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) from neopig import NeoPig -from async_web_fetcher import CrawlMode +from neopig.async_web_fetcher import CrawlMode async def test_crawl_upload_unturf(): diff --git a/tests/unit/test_async_filevault.py b/tests/unit/test_async_filevault.py index a251a45..dd5f521 100644 --- a/tests/unit/test_async_filevault.py +++ b/tests/unit/test_async_filevault.py @@ -13,7 +13,7 @@ from pathlib import Path sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) -from filevault import AsyncVault, create_async_vault, content_hash +from neopig.filevault import AsyncVault, create_async_vault, content_hash class TestAsyncVaultBasics: diff --git a/tests/unit/test_async_web_fetcher.py b/tests/unit/test_async_web_fetcher.py index b117dcc..566b421 100644 --- a/tests/unit/test_async_web_fetcher.py +++ b/tests/unit/test_async_web_fetcher.py @@ -10,7 +10,7 @@ from unittest.mock import Mock, AsyncMock, patch, MagicMock import asyncio import aiohttp -from async_web_fetcher import ( +from neopig.async_web_fetcher import ( CrawlMode, MediaItem, strip_uri_fragment, diff --git a/tests/unit/test_database.py b/tests/unit/test_database.py index 9a883d5..d14dfd2 100644 --- a/tests/unit/test_database.py +++ b/tests/unit/test_database.py @@ -12,7 +12,7 @@ import os import json from datetime import datetime, timezone -from database import ( +from neopig.database import ( Database, CrawlJob, Media, diff --git a/tests/unit/test_domain_vault.py b/tests/unit/test_domain_vault.py index caef5e0..8971d3b 100644 --- a/tests/unit/test_domain_vault.py +++ b/tests/unit/test_domain_vault.py @@ -12,7 +12,7 @@ import os import hashlib from pathlib import Path -from domain_vault import ( +from neopig.domain_vault import ( GitRepo, DomainHtmlVault, DomainMediaVault, diff --git a/tests/unit/test_filevault.py b/tests/unit/test_filevault.py index d287d36..9a8dc63 100644 --- a/tests/unit/test_filevault.py +++ b/tests/unit/test_filevault.py @@ -14,7 +14,7 @@ from pathlib import Path # Add parent directory to path to import modules sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) -from filevault import ( +from neopig.filevault import ( Vault, create_vault, ensure_bytes, @@ -636,7 +636,7 @@ class TestVersionAndCompatibility: def test_version_info(self): """Test that version information is available""" - import filevault + from neopig import filevault assert hasattr(filevault, "__version__") assert filevault.__version__ == "2.0.0" diff --git a/tests/unit/test_repo.py b/tests/unit/test_repo.py index 1040900..99b184d 100644 --- a/tests/unit/test_repo.py +++ b/tests/unit/test_repo.py @@ -11,7 +11,7 @@ import shutil import os from pathlib import Path -from repo import ( +from neopig.repo import ( detect_vcs, get_repo_path, run_cmd, @@ -206,7 +206,7 @@ class TestCloneRepo: assert success is False assert "Unknown VCS type" in msg - @patch('repo.run_cmd') + @patch('neopig.repo.run_cmd') def test_clone_git_shallow(self, mock_run): """Test git clone with shallow flag.""" mock_run.return_value = (0, "", "") @@ -219,7 +219,7 @@ class TestCloneRepo: assert "--depth" in call_args assert "1" in call_args - @patch('repo.run_cmd') + @patch('neopig.repo.run_cmd') def test_clone_git_full(self, mock_run): """Test git clone without shallow flag.""" mock_run.return_value = (0, "", "") @@ -231,7 +231,7 @@ class TestCloneRepo: call_args = mock_run.call_args[0][0] assert "--depth" not in call_args - @patch('repo.run_cmd') + @patch('neopig.repo.run_cmd') def test_clone_hg(self, mock_run): """Test mercurial clone.""" mock_run.return_value = (0, "", "") @@ -244,7 +244,7 @@ class TestCloneRepo: assert "hg" in call_args assert "clone" in call_args - @patch('repo.run_cmd') + @patch('neopig.repo.run_cmd') def test_clone_svn(self, mock_run): """Test SVN checkout.""" mock_run.return_value = (0, "", "") @@ -262,7 +262,7 @@ class TestCloneRepoAsync: """Test async repository cloning.""" @pytest.mark.asyncio - @patch('repo.clone_repo') + @patch('neopig.repo.clone_repo') async def test_clone_repo_async(self, mock_clone): """Test async clone delegates to sync.""" mock_clone.return_value = (True, "Cloned") @@ -295,7 +295,7 @@ class TestPullRepo: assert success is False assert "No VCS directory found" in msg - @patch('repo.run_cmd') + @patch('neopig.repo.run_cmd') def test_pull_git(self, mock_run): """Test git pull.""" mock_run.return_value = (0, "Already up to date", "") @@ -309,7 +309,7 @@ class TestPullRepo: assert "git" in call_args assert "pull" in call_args - @patch('repo.run_cmd') + @patch('neopig.repo.run_cmd') def test_pull_hg(self, mock_run): """Test mercurial pull.""" mock_run.return_value = (0, "pulling from...", "") @@ -322,7 +322,7 @@ class TestPullRepo: assert "hg" in call_args assert "pull" in call_args - @patch('repo.run_cmd') + @patch('neopig.repo.run_cmd') def test_pull_svn(self, mock_run): """Test SVN update.""" mock_run.return_value = (0, "Updating...", "") @@ -353,7 +353,7 @@ class TestGetCommitHash: result = get_commit_hash(self.repo_path) assert result is None - @patch('repo.run_cmd') + @patch('neopig.repo.run_cmd') def test_get_git_hash(self, mock_run): """Test getting git commit hash.""" mock_run.return_value = (0, "abc123def456\n", "") @@ -363,7 +363,7 @@ class TestGetCommitHash: assert result == "abc123def456" - @patch('repo.run_cmd') + @patch('neopig.repo.run_cmd') def test_get_hg_hash(self, mock_run): """Test getting mercurial changeset hash.""" mock_run.return_value = (0, "abc123+\n", "") diff --git a/tests/unit/test_screenshot.py b/tests/unit/test_screenshot.py index 69cbe5a..2996bbc 100644 --- a/tests/unit/test_screenshot.py +++ b/tests/unit/test_screenshot.py @@ -10,7 +10,7 @@ import asyncio import tempfile import os -from screenshot import ScreenshotConfig, ScreenshotCapture +from neopig.screenshot import ScreenshotConfig, ScreenshotCapture class TestScreenshotConfig: diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py index ff55d24..a50f0ad 100644 --- a/tests/unit/test_storage.py +++ b/tests/unit/test_storage.py @@ -10,7 +10,7 @@ import tempfile import shutil import hashlib -from filevault import AsyncVault, Vault, content_hash, hash_to_path +from neopig.filevault import AsyncVault, Vault, content_hash, hash_to_path class TestAsyncVaultBasics: