Reorganize project structure: move modules to neopig/, scripts to scripts/, data to data/

Makefile

@@ -12,7 +12,7 @@ install: venv
 	$(PIP) install -r requirements.txt
 
 test: install
-	$(PYTHON) test_crawl.py
+	$(PYTHON) -m pytest tests/ -v --tb=short
 
 crawl: install
 	$(PYTHON) neopig.py $(ARGS)
@@ -21,8 +21,8 @@ archive: install
 	$(PYTHON) archive.py $(ARGS)
 
 # Build bootstrap binary for self-extracting archives
-bootstrap: bootstrap.c
-	gcc -O2 -Wall -o bootstrap bootstrap.c -lz
+bootstrap: scripts/bootstrap.c
+	gcc -O2 -Wall -o bootstrap scripts/bootstrap.c -lz
 	@echo "Built: bootstrap ($$(stat -c%s bootstrap 2>/dev/null || stat -f%z bootstrap) bytes)"
 
 # Create self-extracting .run from a tarball

archive.py

@@ -54,9 +54,9 @@ from bs4 import BeautifulSoup
 from tqdm import tqdm
 
 from neopig import NeoPig, setup_logging, rotate_state_file, get_state_file_path
-from async_web_fetcher import CrawlMode
-from screenshot import ScreenshotConfig
-from filevault import hash_to_path
+from neopig.async_web_fetcher import CrawlMode
+from neopig.screenshot import ScreenshotConfig
+from neopig.filevault import hash_to_path
 
 # Optional markdown conversion
 try:

neopig.py

@@ -38,7 +38,7 @@ from typing import List, Dict, Any, Optional, Set, Tuple
 
 from miniuri import Uri
 
-from async_web_fetcher import (
+from neopig.async_web_fetcher import (
     AsyncWebFetcher,
     CrawlMode,
     MediaItem,
@@ -46,11 +46,11 @@ from async_web_fetcher import (
     get_media_type_from_extension,
     get_media_type_from_mime,
 )
-from filevault import AsyncVault, hash_to_path
-from database import Database, SCORE_SCREENSHOT, SCORE_OG_IMAGE, SCORE_THUMBNAIL, SCORE_FULL_RES
-from screenshot import ScreenshotCapture, ScreenshotConfig
-from domain_vault import VaultManager, DomainHtmlVault, DomainMediaVault, DomainLinkpeekVault, extract_media_urls
-from repo import detect_vcs, clone_repo_async, pull_repo_async, get_repo_path, walk_files, get_commit_hash, get_file_language, is_binary_file
+from neopig.filevault import AsyncVault, hash_to_path
+from neopig.database import Database, SCORE_SCREENSHOT, SCORE_OG_IMAGE, SCORE_THUMBNAIL, SCORE_FULL_RES
+from neopig.screenshot import ScreenshotCapture, ScreenshotConfig
+from neopig.domain_vault import VaultManager, DomainHtmlVault, DomainMediaVault, DomainLinkpeekVault, extract_media_urls
+from neopig.repo import detect_vcs, clone_repo_async, pull_repo_async, get_repo_path, walk_files, get_commit_hash, get_file_language, is_binary_file
 from tqdm import tqdm
 
 logger = logging.getLogger(__name__)

neopig/async_filevault.py

@@ -19,7 +19,7 @@ import asyncio
 from functools import partial
 from typing import Any, Optional, List
 
-from filevault import Vault, ensure_bytes, create_vault
+from .filevault import Vault, ensure_bytes, create_vault
 
 __version__ = "1.1.0"
 __author__ = "Russell Ballestrini"

serp/server.py

@@ -30,8 +30,8 @@ from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, Redirect
 from sqlalchemy import text
 import uvicorn
 
-from database import Database, OVER_9000
-from filevault import hash_to_path
+from neopig.database import Database, OVER_9000
+from neopig.filevault import hash_to_path
 from miniuri import Uri
 from neopig.live import get_live_queue
 
@@ -903,7 +903,7 @@ async def get_stats_endpoint():
     # Add page count (handled separately since table may not exist)
     try:
         from sqlalchemy import select, func
-        from database import Page
+        from neopig.database import Page
         async with db.session() as session:
             result = await session.execute(select(func.count()).select_from(Page))
             stats['total_pages'] = result.scalar() or 0
@@ -916,7 +916,7 @@ async def get_stats_endpoint():
 async def random_item(type: str = Query(None, description="Type: media or page (random if not specified)")):
     """Redirect to a random media item or page."""
     from sqlalchemy import select, func
-    from database import Media, Page
+    from neopig.database import Media, Page
     import random
 
     # If no type specified, randomly pick between media and page

tests/test_crawl.py

@@ -13,7 +13,7 @@ from pathlib import Path
 sys.path.insert(0, str(Path(__file__).parent))
 
 from neopig import NeoPig
-from async_web_fetcher import CrawlMode
+from neopig.async_web_fetcher import CrawlMode
 
 
 async def test_crawl_upload_unturf():

tests/unit/test_async_filevault.py

@@ -13,7 +13,7 @@ from pathlib import Path
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
 
-from filevault import AsyncVault, create_async_vault, content_hash
+from neopig.filevault import AsyncVault, create_async_vault, content_hash
 
 
 class TestAsyncVaultBasics:

tests/unit/test_async_web_fetcher.py

@@ -10,7 +10,7 @@ from unittest.mock import Mock, AsyncMock, patch, MagicMock
 import asyncio
 import aiohttp
 
-from async_web_fetcher import (
+from neopig.async_web_fetcher import (
     CrawlMode,
     MediaItem,
     strip_uri_fragment,

tests/unit/test_database.py

@@ -12,7 +12,7 @@ import os
 import json
 from datetime import datetime, timezone
 
-from database import (
+from neopig.database import (
     Database,
     CrawlJob,
     Media,

tests/unit/test_domain_vault.py

@@ -12,7 +12,7 @@ import os
 import hashlib
 from pathlib import Path
 
-from domain_vault import (
+from neopig.domain_vault import (
     GitRepo,
     DomainHtmlVault,
     DomainMediaVault,

tests/unit/test_filevault.py

@@ -14,7 +14,7 @@ from pathlib import Path
 # Add parent directory to path to import modules
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
 
-from filevault import (
+from neopig.filevault import (
     Vault,
     create_vault,
     ensure_bytes,
@@ -636,7 +636,7 @@ class TestVersionAndCompatibility:
 
     def test_version_info(self):
         """Test that version information is available"""
-        import filevault
+        from neopig import filevault
 
         assert hasattr(filevault, "__version__")
         assert filevault.__version__ == "2.0.0"

tests/unit/test_repo.py

@@ -11,7 +11,7 @@ import shutil
 import os
 from pathlib import Path
 
-from repo import (
+from neopig.repo import (
     detect_vcs,
     get_repo_path,
     run_cmd,
@@ -206,7 +206,7 @@ class TestCloneRepo:
         assert success is False
         assert "Unknown VCS type" in msg
 
-    @patch('repo.run_cmd')
+    @patch('neopig.repo.run_cmd')
     def test_clone_git_shallow(self, mock_run):
         """Test git clone with shallow flag."""
         mock_run.return_value = (0, "", "")
@@ -219,7 +219,7 @@ class TestCloneRepo:
         assert "--depth" in call_args
         assert "1" in call_args
 
-    @patch('repo.run_cmd')
+    @patch('neopig.repo.run_cmd')
     def test_clone_git_full(self, mock_run):
         """Test git clone without shallow flag."""
         mock_run.return_value = (0, "", "")
@@ -231,7 +231,7 @@ class TestCloneRepo:
         call_args = mock_run.call_args[0][0]
         assert "--depth" not in call_args
 
-    @patch('repo.run_cmd')
+    @patch('neopig.repo.run_cmd')
     def test_clone_hg(self, mock_run):
         """Test mercurial clone."""
         mock_run.return_value = (0, "", "")
@@ -244,7 +244,7 @@ class TestCloneRepo:
         assert "hg" in call_args
         assert "clone" in call_args
 
-    @patch('repo.run_cmd')
+    @patch('neopig.repo.run_cmd')
     def test_clone_svn(self, mock_run):
         """Test SVN checkout."""
         mock_run.return_value = (0, "", "")
@@ -262,7 +262,7 @@ class TestCloneRepoAsync:
     """Test async repository cloning."""
 
     @pytest.mark.asyncio
-    @patch('repo.clone_repo')
+    @patch('neopig.repo.clone_repo')
     async def test_clone_repo_async(self, mock_clone):
         """Test async clone delegates to sync."""
         mock_clone.return_value = (True, "Cloned")
@@ -295,7 +295,7 @@ class TestPullRepo:
         assert success is False
         assert "No VCS directory found" in msg
 
-    @patch('repo.run_cmd')
+    @patch('neopig.repo.run_cmd')
     def test_pull_git(self, mock_run):
         """Test git pull."""
         mock_run.return_value = (0, "Already up to date", "")
@@ -309,7 +309,7 @@ class TestPullRepo:
         assert "git" in call_args
         assert "pull" in call_args
 
-    @patch('repo.run_cmd')
+    @patch('neopig.repo.run_cmd')
     def test_pull_hg(self, mock_run):
         """Test mercurial pull."""
         mock_run.return_value = (0, "pulling from...", "")
@@ -322,7 +322,7 @@ class TestPullRepo:
         assert "hg" in call_args
         assert "pull" in call_args
 
-    @patch('repo.run_cmd')
+    @patch('neopig.repo.run_cmd')
     def test_pull_svn(self, mock_run):
         """Test SVN update."""
         mock_run.return_value = (0, "Updating...", "")
@@ -353,7 +353,7 @@ class TestGetCommitHash:
         result = get_commit_hash(self.repo_path)
         assert result is None
 
-    @patch('repo.run_cmd')
+    @patch('neopig.repo.run_cmd')
     def test_get_git_hash(self, mock_run):
         """Test getting git commit hash."""
         mock_run.return_value = (0, "abc123def456\n", "")
@@ -363,7 +363,7 @@ class TestGetCommitHash:
 
         assert result == "abc123def456"
 
-    @patch('repo.run_cmd')
+    @patch('neopig.repo.run_cmd')
     def test_get_hg_hash(self, mock_run):
         """Test getting mercurial changeset hash."""
         mock_run.return_value = (0, "abc123+\n", "")

tests/unit/test_screenshot.py

@@ -10,7 +10,7 @@ import asyncio
 import tempfile
 import os
 
-from screenshot import ScreenshotConfig, ScreenshotCapture
+from neopig.screenshot import ScreenshotConfig, ScreenshotCapture
 
 
 class TestScreenshotConfig:

tests/unit/test_storage.py

@@ -10,7 +10,7 @@ import tempfile
 import shutil
 import hashlib
 
-from filevault import AsyncVault, Vault, content_hash, hash_to_path
+from neopig.filevault import AsyncVault, Vault, content_hash, hash_to_path
 
 
 class TestAsyncVaultBasics: