Add comprehensive unit tests for async modules
Tests for: - async_web_fetcher: URL handling, media extraction, crawl modes, scoring - domain_vault: VaultManager, HTML/Media/Linkpeek vaults - screenshot: ScreenshotCapture configuration and availability - storage: ImageVault with MD5 deduplication 185 tests total, all passing.
This commit is contained in:
parent
54087c5bb8
commit
1847a2fee6
6 changed files with 1486 additions and 0 deletions
7
pytest.ini
Normal file
7
pytest.ini
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
[pytest]
|
||||
asyncio_mode = auto
|
||||
asyncio_default_fixture_loop_scope = function
|
||||
testpaths = tests
|
||||
python_files = test_*.py
|
||||
python_classes = Test*
|
||||
python_functions = test_*
|
||||
28
tests/conftest.py
Normal file
28
tests/conftest.py
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
"""
|
||||
Pytest configuration for neopig tests.
|
||||
Configures pytest-asyncio for async test support.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
# Configure pytest-asyncio to use auto mode
|
||||
pytest_plugins = ('pytest_asyncio',)
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Configure pytest-asyncio mode."""
|
||||
config.addinivalue_line(
|
||||
"markers", "asyncio: mark test as async"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def event_loop_policy():
|
||||
"""Use default event loop policy."""
|
||||
import asyncio
|
||||
return asyncio.DefaultEventLoopPolicy()
|
||||
458
tests/unit/test_async_web_fetcher.py
Normal file
458
tests/unit/test_async_web_fetcher.py
Normal file
|
|
@ -0,0 +1,458 @@
|
|||
"""
|
||||
Tests for async_web_fetcher module.
|
||||
|
||||
Tests URL handling, media extraction, robots.txt compliance,
|
||||
and async fetching functionality.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, AsyncMock, patch, MagicMock
|
||||
import asyncio
|
||||
import aiohttp
|
||||
|
||||
from async_web_fetcher import (
|
||||
CrawlMode,
|
||||
MediaItem,
|
||||
strip_uri_fragment,
|
||||
normalize_link,
|
||||
get_media_type_from_extension,
|
||||
get_media_type_from_mime,
|
||||
extract_media_from_html,
|
||||
AsyncWebFetcher,
|
||||
IMAGE_EXTENSIONS,
|
||||
VIDEO_EXTENSIONS,
|
||||
AUDIO_EXTENSIONS,
|
||||
DEFAULT_CRAWL_DELAY,
|
||||
get_last_fetch_error,
|
||||
clear_last_fetch_error,
|
||||
)
|
||||
|
||||
|
||||
class TestCrawlMode:
|
||||
"""Test CrawlMode enum."""
|
||||
|
||||
def test_crawl_mode_values(self):
|
||||
"""Test all crawl mode values exist."""
|
||||
assert CrawlMode.TEXT.value == "text"
|
||||
assert CrawlMode.IMAGES.value == "images"
|
||||
assert CrawlMode.VIDEOS.value == "videos"
|
||||
assert CrawlMode.MEDIA.value == "media"
|
||||
assert CrawlMode.ALL.value == "all"
|
||||
|
||||
def test_crawl_mode_from_string(self):
|
||||
"""Test creating CrawlMode from string."""
|
||||
assert CrawlMode("text") == CrawlMode.TEXT
|
||||
assert CrawlMode("images") == CrawlMode.IMAGES
|
||||
assert CrawlMode("media") == CrawlMode.MEDIA
|
||||
|
||||
|
||||
class TestMediaItem:
|
||||
"""Test MediaItem dataclass."""
|
||||
|
||||
def test_media_item_creation(self):
|
||||
"""Test creating a MediaItem."""
|
||||
item = MediaItem(
|
||||
url="https://example.com/image.jpg",
|
||||
source_page="https://example.com/",
|
||||
media_type="image",
|
||||
mime_type="image/jpeg",
|
||||
)
|
||||
assert item.url == "https://example.com/image.jpg"
|
||||
assert item.source_page == "https://example.com/"
|
||||
assert item.media_type == "image"
|
||||
assert item.mime_type == "image/jpeg"
|
||||
assert item.discovered_at is not None
|
||||
|
||||
def test_media_item_optional_fields(self):
|
||||
"""Test MediaItem with optional fields."""
|
||||
item = MediaItem(
|
||||
url="https://example.com/video.mp4",
|
||||
source_page="https://example.com/",
|
||||
media_type="video",
|
||||
alt_text="A video",
|
||||
width=1920,
|
||||
height=1080,
|
||||
)
|
||||
assert item.alt_text == "A video"
|
||||
assert item.width == 1920
|
||||
assert item.height == 1080
|
||||
|
||||
|
||||
class TestURIHandling:
|
||||
"""Test URI handling functions."""
|
||||
|
||||
def test_strip_uri_fragment_basic(self):
|
||||
"""Test stripping fragment from URI."""
|
||||
uri = "https://example.com/page#section"
|
||||
result = strip_uri_fragment(uri)
|
||||
assert result == "https://example.com/page"
|
||||
|
||||
def test_strip_uri_fragment_no_fragment(self):
|
||||
"""Test URI without fragment."""
|
||||
uri = "https://example.com/page"
|
||||
result = strip_uri_fragment(uri)
|
||||
assert result == "https://example.com/page"
|
||||
|
||||
def test_strip_uri_fragment_with_query(self):
|
||||
"""Test URI with query and fragment."""
|
||||
uri = "https://example.com/page?q=test#section"
|
||||
result = strip_uri_fragment(uri)
|
||||
assert result == "https://example.com/page?q=test"
|
||||
|
||||
def test_strip_uri_fragment_empty(self):
|
||||
"""Test empty URI."""
|
||||
assert strip_uri_fragment("") == ""
|
||||
assert strip_uri_fragment(None) is None
|
||||
|
||||
def test_normalize_link_dict(self):
|
||||
"""Test normalizing link from dict."""
|
||||
link = {"url": "https://example.com/page#section", "anchor_text": "Click here"}
|
||||
uri, anchor = normalize_link(link)
|
||||
assert uri == "https://example.com/page"
|
||||
assert anchor == "Click here"
|
||||
|
||||
def test_normalize_link_string(self):
|
||||
"""Test normalizing link from string."""
|
||||
uri, anchor = normalize_link("https://example.com/page#section")
|
||||
assert uri == "https://example.com/page"
|
||||
assert anchor == ""
|
||||
|
||||
|
||||
class TestMediaTypeDetection:
|
||||
"""Test media type detection functions."""
|
||||
|
||||
def test_get_media_type_from_extension_image(self):
|
||||
"""Test detecting image from extension."""
|
||||
assert get_media_type_from_extension("https://example.com/photo.jpg") == "image"
|
||||
assert get_media_type_from_extension("https://example.com/photo.png") == "image"
|
||||
assert get_media_type_from_extension("https://example.com/photo.webp") == "image"
|
||||
assert get_media_type_from_extension("https://example.com/photo.gif") == "image"
|
||||
|
||||
def test_get_media_type_from_extension_video(self):
|
||||
"""Test detecting video from extension."""
|
||||
assert get_media_type_from_extension("https://example.com/video.mp4") == "video"
|
||||
assert get_media_type_from_extension("https://example.com/video.webm") == "video"
|
||||
assert get_media_type_from_extension("https://example.com/video.mov") == "video"
|
||||
|
||||
def test_get_media_type_from_extension_audio(self):
|
||||
"""Test detecting audio from extension."""
|
||||
assert get_media_type_from_extension("https://example.com/audio.mp3") == "audio"
|
||||
assert get_media_type_from_extension("https://example.com/audio.wav") == "audio"
|
||||
assert get_media_type_from_extension("https://example.com/audio.ogg") == "audio"
|
||||
|
||||
def test_get_media_type_from_extension_unknown(self):
|
||||
"""Test unknown extension."""
|
||||
assert get_media_type_from_extension("https://example.com/file.txt") is None
|
||||
assert get_media_type_from_extension("https://example.com/page.html") is None
|
||||
|
||||
def test_get_media_type_from_mime_image(self):
|
||||
"""Test detecting image from MIME type."""
|
||||
assert get_media_type_from_mime("image/jpeg") == "image"
|
||||
assert get_media_type_from_mime("image/png") == "image"
|
||||
assert get_media_type_from_mime("image/webp") == "image"
|
||||
|
||||
def test_get_media_type_from_mime_video(self):
|
||||
"""Test detecting video from MIME type."""
|
||||
assert get_media_type_from_mime("video/mp4") == "video"
|
||||
assert get_media_type_from_mime("video/webm") == "video"
|
||||
|
||||
def test_get_media_type_from_mime_audio(self):
|
||||
"""Test detecting audio from MIME type."""
|
||||
assert get_media_type_from_mime("audio/mpeg") == "audio"
|
||||
assert get_media_type_from_mime("audio/wav") == "audio"
|
||||
|
||||
def test_get_media_type_from_mime_unknown(self):
|
||||
"""Test unknown MIME type."""
|
||||
assert get_media_type_from_mime("text/html") is None
|
||||
assert get_media_type_from_mime("application/json") is None
|
||||
|
||||
|
||||
class TestMediaExtraction:
|
||||
"""Test media extraction from HTML."""
|
||||
|
||||
def test_extract_images_from_html(self):
|
||||
"""Test extracting images from HTML."""
|
||||
html = '''
|
||||
<html>
|
||||
<body>
|
||||
<img src="/images/photo.jpg" alt="A photo">
|
||||
<img src="https://cdn.example.com/logo.png" alt="Logo">
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
media = extract_media_from_html(html, "https://example.com/", CrawlMode.IMAGES)
|
||||
|
||||
assert len(media) >= 1
|
||||
urls = [m['url'] for m in media]
|
||||
# Should have resolved relative URL
|
||||
assert any('photo.jpg' in url for url in urls)
|
||||
|
||||
def test_extract_videos_from_html(self):
|
||||
"""Test extracting videos from HTML."""
|
||||
html = '''
|
||||
<html>
|
||||
<body>
|
||||
<video src="/videos/movie.mp4"></video>
|
||||
<source src="/videos/clip.webm" type="video/webm">
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
media = extract_media_from_html(html, "https://example.com/", CrawlMode.VIDEOS)
|
||||
|
||||
urls = [m['url'] for m in media]
|
||||
assert any('movie.mp4' in url or 'clip.webm' in url for url in urls)
|
||||
|
||||
def test_extract_media_filters_by_mode(self):
|
||||
"""Test that extraction respects crawl mode."""
|
||||
html = '''
|
||||
<html>
|
||||
<body>
|
||||
<img src="/photo.jpg">
|
||||
<video src="/video.mp4"></video>
|
||||
<audio src="/audio.mp3"></audio>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
|
||||
# Images only
|
||||
images = extract_media_from_html(html, "https://example.com/", CrawlMode.IMAGES)
|
||||
image_urls = [m['url'] for m in images]
|
||||
assert any('photo.jpg' in url for url in image_urls)
|
||||
assert not any('video.mp4' in url for url in image_urls)
|
||||
|
||||
# Videos only
|
||||
videos = extract_media_from_html(html, "https://example.com/", CrawlMode.VIDEOS)
|
||||
video_urls = [m['url'] for m in videos]
|
||||
assert any('video.mp4' in url for url in video_urls)
|
||||
|
||||
|
||||
class TestAsyncWebFetcher:
|
||||
"""Test AsyncWebFetcher class."""
|
||||
|
||||
def test_fetcher_initialization(self):
|
||||
"""Test AsyncWebFetcher initialization."""
|
||||
fetcher = AsyncWebFetcher(
|
||||
user_agent="TestBot/1.0",
|
||||
default_crawl_delay=1.0,
|
||||
)
|
||||
assert fetcher.user_agent == "TestBot/1.0"
|
||||
assert fetcher.default_crawl_delay == 1.0
|
||||
|
||||
def test_fetcher_default_values(self):
|
||||
"""Test AsyncWebFetcher default values."""
|
||||
fetcher = AsyncWebFetcher()
|
||||
assert fetcher.default_crawl_delay == DEFAULT_CRAWL_DELAY
|
||||
|
||||
def test_get_domain(self):
|
||||
"""Test domain extraction."""
|
||||
fetcher = AsyncWebFetcher()
|
||||
assert fetcher._get_domain("https://example.com/page") == "example.com"
|
||||
assert fetcher._get_domain("https://sub.example.com/page") == "sub.example.com"
|
||||
|
||||
def test_simple_stem(self):
|
||||
"""Test simple word stemming."""
|
||||
fetcher = AsyncWebFetcher()
|
||||
|
||||
# Test stemming - actual behavior may vary
|
||||
stem = fetcher._simple_stem("running")
|
||||
assert "run" in stem # Should contain "run"
|
||||
|
||||
# Short words should not be stemmed
|
||||
assert fetcher._simple_stem("go") == "go"
|
||||
|
||||
def test_extract_text_content(self):
|
||||
"""Test text extraction from HTML."""
|
||||
fetcher = AsyncWebFetcher()
|
||||
html = '''
|
||||
<html>
|
||||
<head><title>Test Page</title></head>
|
||||
<body>
|
||||
<h1>Hello World</h1>
|
||||
<p>This is a test paragraph.</p>
|
||||
<script>alert('ignored');</script>
|
||||
<style>.hidden { display: none; }</style>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
text = fetcher.extract_text_content(html)
|
||||
|
||||
assert "Hello World" in text
|
||||
assert "test paragraph" in text
|
||||
assert "alert" not in text # Script content should be removed
|
||||
assert ".hidden" not in text # Style content should be removed
|
||||
|
||||
def test_extract_title(self):
|
||||
"""Test title extraction from HTML."""
|
||||
fetcher = AsyncWebFetcher()
|
||||
html = '<html><head><title>Page Title</title></head><body></body></html>'
|
||||
title = fetcher.extract_title(html)
|
||||
assert title == "Page Title"
|
||||
|
||||
def test_extract_title_missing(self):
|
||||
"""Test title extraction when missing."""
|
||||
fetcher = AsyncWebFetcher()
|
||||
html = '<html><head></head><body></body></html>'
|
||||
title = fetcher.extract_title(html)
|
||||
# Returns "Untitled" when no title present
|
||||
assert title == "Untitled" or title == ""
|
||||
|
||||
|
||||
class TestAsyncWebFetcherScoring:
|
||||
"""Test link and page scoring functionality."""
|
||||
|
||||
def test_score_link_keyword_match(self):
|
||||
"""Test link scoring with keyword matches."""
|
||||
fetcher = AsyncWebFetcher()
|
||||
|
||||
# Link with keyword in URL
|
||||
score = fetcher._score_link(
|
||||
"https://example.com/pricing",
|
||||
"View Pricing",
|
||||
query_keywords=["pricing", "cost"],
|
||||
)
|
||||
assert score > 0 # Should have positive score
|
||||
|
||||
# Link without keywords
|
||||
score_none = fetcher._score_link(
|
||||
"https://example.com/about",
|
||||
"About Us",
|
||||
query_keywords=["pricing", "cost"],
|
||||
)
|
||||
assert score > score_none # Keyword match should score higher
|
||||
|
||||
def test_score_link_anchor_text(self):
|
||||
"""Test link scoring with anchor text."""
|
||||
fetcher = AsyncWebFetcher()
|
||||
|
||||
# Anchor text with keyword
|
||||
score = fetcher._score_link(
|
||||
"https://example.com/page",
|
||||
"Our Pricing Plans",
|
||||
query_keywords=["pricing"],
|
||||
)
|
||||
assert score > 0
|
||||
|
||||
def test_score_page_keyword_match(self):
|
||||
"""Test page scoring with keyword matches."""
|
||||
fetcher = AsyncWebFetcher()
|
||||
|
||||
page_data = {
|
||||
"url": "https://example.com/pricing",
|
||||
"title": "Pricing Plans",
|
||||
"extracted_text": "Our pricing is competitive. See our cost breakdown.",
|
||||
}
|
||||
|
||||
score = fetcher._score_page(
|
||||
page_data,
|
||||
query_keywords=["pricing", "cost"],
|
||||
)
|
||||
assert score > 0
|
||||
|
||||
|
||||
class TestAsyncWebFetcherAsync:
|
||||
"""Test async methods of AsyncWebFetcher."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_can_fetch_respects_robots(self):
|
||||
"""Test that _can_fetch respects robots.txt."""
|
||||
fetcher = AsyncWebFetcher()
|
||||
|
||||
# Mock the robots.txt fetch to return a parser that disallows
|
||||
with patch.object(fetcher, '_fetch_robots_txt', new_callable=AsyncMock) as mock_robots:
|
||||
mock_parser = MagicMock()
|
||||
mock_parser.can_fetch.return_value = False
|
||||
mock_robots.return_value = mock_parser
|
||||
|
||||
result = await fetcher._can_fetch("https://example.com/blocked")
|
||||
assert result is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_can_fetch_allows_when_parser_allows(self):
|
||||
"""Test _can_fetch when robots parser allows."""
|
||||
fetcher = AsyncWebFetcher()
|
||||
|
||||
with patch.object(fetcher, '_fetch_robots_txt', new_callable=AsyncMock) as mock_robots:
|
||||
mock_parser = MagicMock()
|
||||
mock_parser.can_fetch.return_value = True
|
||||
mock_robots.return_value = mock_parser
|
||||
|
||||
result = await fetcher._can_fetch("https://example.com/allowed")
|
||||
assert result is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enforce_crawl_delay(self):
|
||||
"""Test crawl delay enforcement."""
|
||||
fetcher = AsyncWebFetcher(default_crawl_delay=0.1)
|
||||
domain = "example.com"
|
||||
|
||||
# First request should not delay
|
||||
start = asyncio.get_event_loop().time()
|
||||
await fetcher._enforce_crawl_delay(domain)
|
||||
first_time = asyncio.get_event_loop().time() - start
|
||||
|
||||
# Second request should delay
|
||||
start = asyncio.get_event_loop().time()
|
||||
await fetcher._enforce_crawl_delay(domain)
|
||||
second_time = asyncio.get_event_loop().time() - start
|
||||
|
||||
# Second call should have some delay (at least part of 0.1s)
|
||||
assert second_time >= 0.05 # Allow some tolerance
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_media_url_signature(self):
|
||||
"""Test check_media_url function signature."""
|
||||
fetcher = AsyncWebFetcher()
|
||||
|
||||
# Verify the method exists and has correct signature
|
||||
import inspect
|
||||
sig = inspect.signature(fetcher.check_media_url)
|
||||
params = list(sig.parameters.keys())
|
||||
|
||||
# Should have url parameter
|
||||
assert 'url' in params or len(params) >= 1
|
||||
|
||||
|
||||
class TestErrorHandling:
|
||||
"""Test error handling functionality."""
|
||||
|
||||
def test_get_last_fetch_error_initially_none(self):
|
||||
"""Test that last fetch error is None initially."""
|
||||
clear_last_fetch_error()
|
||||
error = get_last_fetch_error()
|
||||
assert error is None
|
||||
|
||||
def test_clear_last_fetch_error(self):
|
||||
"""Test clearing the last fetch error."""
|
||||
clear_last_fetch_error()
|
||||
assert get_last_fetch_error() is None
|
||||
|
||||
|
||||
class TestExtensionSets:
|
||||
"""Test extension and MIME type sets."""
|
||||
|
||||
def test_image_extensions(self):
|
||||
"""Test image extension set."""
|
||||
assert '.jpg' in IMAGE_EXTENSIONS
|
||||
assert '.jpeg' in IMAGE_EXTENSIONS
|
||||
assert '.png' in IMAGE_EXTENSIONS
|
||||
assert '.gif' in IMAGE_EXTENSIONS
|
||||
assert '.webp' in IMAGE_EXTENSIONS
|
||||
assert '.svg' in IMAGE_EXTENSIONS
|
||||
|
||||
def test_video_extensions(self):
|
||||
"""Test video extension set."""
|
||||
assert '.mp4' in VIDEO_EXTENSIONS
|
||||
assert '.webm' in VIDEO_EXTENSIONS
|
||||
assert '.mov' in VIDEO_EXTENSIONS
|
||||
assert '.avi' in VIDEO_EXTENSIONS
|
||||
|
||||
def test_audio_extensions(self):
|
||||
"""Test audio extension set."""
|
||||
assert '.mp3' in AUDIO_EXTENSIONS
|
||||
assert '.wav' in AUDIO_EXTENSIONS
|
||||
assert '.ogg' in AUDIO_EXTENSIONS
|
||||
assert '.flac' in AUDIO_EXTENSIONS
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
364
tests/unit/test_domain_vault.py
Normal file
364
tests/unit/test_domain_vault.py
Normal file
|
|
@ -0,0 +1,364 @@
|
|||
"""
|
||||
Tests for domain_vault module.
|
||||
|
||||
Tests domain-based vault system for HTML, media, and screenshots.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, AsyncMock, patch, MagicMock
|
||||
import tempfile
|
||||
import shutil
|
||||
import os
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from domain_vault import (
|
||||
GitRepo,
|
||||
DomainHtmlVault,
|
||||
DomainMediaVault,
|
||||
DomainLinkpeekVault,
|
||||
VaultManager,
|
||||
)
|
||||
|
||||
|
||||
class TestGitRepo:
|
||||
"""Test GitRepo class."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.repo_path = Path(self.temp_dir) / "test_repo"
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_git_repo_initialization(self):
|
||||
"""Test GitRepo initialization."""
|
||||
repo = GitRepo(self.repo_path)
|
||||
|
||||
assert repo.path == self.repo_path
|
||||
assert repo.use_lfs is False
|
||||
|
||||
def test_git_repo_with_lfs(self):
|
||||
"""Test GitRepo with LFS enabled."""
|
||||
repo = GitRepo(self.repo_path, use_lfs=True)
|
||||
|
||||
assert repo.use_lfs is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_git_repo_init_creates_directory(self):
|
||||
"""Test that init creates the directory."""
|
||||
repo = GitRepo(self.repo_path)
|
||||
|
||||
with patch.object(repo, '_run', new_callable=AsyncMock) as mock_run:
|
||||
mock_run.return_value = (b'', b'')
|
||||
await repo.init()
|
||||
|
||||
assert self.repo_path.exists()
|
||||
|
||||
|
||||
class TestDomainHtmlVault:
|
||||
"""Test DomainHtmlVault class."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.vault_base = os.path.join(self.temp_dir, "vault")
|
||||
self.domain = "example.com"
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_html_vault_initialization(self):
|
||||
"""Test DomainHtmlVault initialization."""
|
||||
vault = DomainHtmlVault(self.vault_base, self.domain)
|
||||
|
||||
assert vault.domain == self.domain.lower()
|
||||
assert vault.media_base_url == '/media'
|
||||
|
||||
def test_html_vault_media_base_url(self):
|
||||
"""Test custom media base URL."""
|
||||
vault = DomainHtmlVault(
|
||||
self.vault_base,
|
||||
self.domain,
|
||||
media_base_url="/assets"
|
||||
)
|
||||
|
||||
assert vault.media_base_url == "/assets"
|
||||
|
||||
def test_get_media_vault_url(self):
|
||||
"""Test media URL rewriting."""
|
||||
vault = DomainHtmlVault(self.vault_base, self.domain)
|
||||
|
||||
media_url = "https://example.com/images/photo.jpg"
|
||||
result = vault.get_media_vault_url(media_url)
|
||||
|
||||
assert vault.media_base_url in result
|
||||
assert "photo.jpg" in result or "images" in result
|
||||
|
||||
def test_rewrite_media_urls(self):
|
||||
"""Test HTML media URL rewriting."""
|
||||
vault = DomainHtmlVault(self.vault_base, self.domain)
|
||||
|
||||
html = '<img src="https://example.com/photo.jpg">'
|
||||
mappings = {"https://example.com/photo.jpg": "/media/abc123.jpg"}
|
||||
|
||||
result = vault.rewrite_media_urls(html, "https://example.com/", mappings)
|
||||
|
||||
# Should contain the rewritten URL
|
||||
assert "/media/abc123.jpg" in result
|
||||
|
||||
|
||||
class TestDomainMediaVault:
|
||||
"""Test DomainMediaVault class."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.vault_base = os.path.join(self.temp_dir, "vault")
|
||||
self.domain = "example.com"
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_media_vault_initialization(self):
|
||||
"""Test DomainMediaVault initialization."""
|
||||
vault = DomainMediaVault(self.vault_base, self.domain)
|
||||
|
||||
assert vault.domain == self.domain.lower()
|
||||
|
||||
|
||||
class TestDomainLinkpeekVault:
|
||||
"""Test DomainLinkpeekVault class."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.vault_base = os.path.join(self.temp_dir, "vault")
|
||||
self.domain = "example.com"
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_linkpeek_vault_initialization(self):
|
||||
"""Test DomainLinkpeekVault initialization."""
|
||||
vault = DomainLinkpeekVault(self.vault_base, self.domain)
|
||||
|
||||
assert vault.domain == self.domain.lower()
|
||||
|
||||
def test_url_to_screenshot_path(self):
|
||||
"""Test URL to screenshot path conversion."""
|
||||
vault = DomainLinkpeekVault(self.vault_base, self.domain)
|
||||
|
||||
url = "https://example.com/page"
|
||||
path = vault.url_to_screenshot_path(url)
|
||||
|
||||
# Should generate a path ending in .png
|
||||
assert path.endswith('.png')
|
||||
|
||||
|
||||
class TestVaultManager:
|
||||
"""Test VaultManager class."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.html_base = os.path.join(self.temp_dir, "html_vault")
|
||||
self.media_base = os.path.join(self.temp_dir, "media_vault")
|
||||
self.linkpeek_base = os.path.join(self.temp_dir, "linkpeek_vault")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_vault_manager_initialization(self):
|
||||
"""Test VaultManager initialization."""
|
||||
manager = VaultManager(
|
||||
html_vault_base=self.html_base,
|
||||
media_vault_base=self.media_base,
|
||||
linkpeek_vault_base=self.linkpeek_base,
|
||||
)
|
||||
|
||||
assert str(manager.html_base) == self.html_base
|
||||
|
||||
def test_vault_manager_default_init(self):
|
||||
"""Test VaultManager with defaults."""
|
||||
manager = VaultManager()
|
||||
|
||||
assert manager.html_base == Path('html_vault')
|
||||
assert manager.media_base == Path('media_vault')
|
||||
assert manager.linkpeek_base == Path('linkpeek_vault')
|
||||
|
||||
def test_get_html_vault(self):
|
||||
"""Test getting HTML vault for domain."""
|
||||
manager = VaultManager(
|
||||
html_vault_base=self.html_base,
|
||||
media_vault_base=self.media_base,
|
||||
linkpeek_vault_base=self.linkpeek_base,
|
||||
)
|
||||
|
||||
vault = manager.get_html_vault("example.com")
|
||||
|
||||
assert isinstance(vault, DomainHtmlVault)
|
||||
assert vault.domain == "example.com"
|
||||
|
||||
def test_get_html_vault_caches(self):
|
||||
"""Test that HTML vaults are cached."""
|
||||
manager = VaultManager(
|
||||
html_vault_base=self.html_base,
|
||||
media_vault_base=self.media_base,
|
||||
linkpeek_vault_base=self.linkpeek_base,
|
||||
)
|
||||
|
||||
vault1 = manager.get_html_vault("example.com")
|
||||
vault2 = manager.get_html_vault("example.com")
|
||||
|
||||
assert vault1 is vault2
|
||||
|
||||
def test_get_html_vault_case_insensitive(self):
|
||||
"""Test that domain lookup is case insensitive."""
|
||||
manager = VaultManager(
|
||||
html_vault_base=self.html_base,
|
||||
media_vault_base=self.media_base,
|
||||
linkpeek_vault_base=self.linkpeek_base,
|
||||
)
|
||||
|
||||
vault1 = manager.get_html_vault("Example.COM")
|
||||
vault2 = manager.get_html_vault("example.com")
|
||||
|
||||
assert vault1 is vault2
|
||||
assert vault1.domain == "example.com"
|
||||
|
||||
def test_get_media_vault(self):
|
||||
"""Test getting media vault for domain."""
|
||||
manager = VaultManager(
|
||||
html_vault_base=self.html_base,
|
||||
media_vault_base=self.media_base,
|
||||
linkpeek_vault_base=self.linkpeek_base,
|
||||
)
|
||||
|
||||
vault = manager.get_media_vault("example.com")
|
||||
|
||||
assert isinstance(vault, DomainMediaVault)
|
||||
assert vault.domain == "example.com"
|
||||
|
||||
def test_get_linkpeek_vault(self):
|
||||
"""Test getting linkpeek vault for domain."""
|
||||
manager = VaultManager(
|
||||
html_vault_base=self.html_base,
|
||||
media_vault_base=self.media_base,
|
||||
linkpeek_vault_base=self.linkpeek_base,
|
||||
)
|
||||
|
||||
vault = manager.get_linkpeek_vault("example.com")
|
||||
|
||||
assert isinstance(vault, DomainLinkpeekVault)
|
||||
assert vault.domain == "example.com"
|
||||
|
||||
def test_get_vaults_for_url(self):
|
||||
"""Test getting all vaults for a URL."""
|
||||
manager = VaultManager(
|
||||
html_vault_base=self.html_base,
|
||||
media_vault_base=self.media_base,
|
||||
linkpeek_vault_base=self.linkpeek_base,
|
||||
)
|
||||
|
||||
html_vault, media_vault, linkpeek_vault = manager.get_vaults_for_url(
|
||||
"https://example.com/page"
|
||||
)
|
||||
|
||||
assert isinstance(html_vault, DomainHtmlVault)
|
||||
assert isinstance(media_vault, DomainMediaVault)
|
||||
assert isinstance(linkpeek_vault, DomainLinkpeekVault)
|
||||
assert html_vault.domain == "example.com"
|
||||
|
||||
def test_get_linkpeek_url(self):
|
||||
"""Test generating linkpeek URL."""
|
||||
manager = VaultManager(
|
||||
html_vault_base=self.html_base,
|
||||
media_vault_base=self.media_base,
|
||||
linkpeek_vault_base=self.linkpeek_base,
|
||||
linkpeek_base_url="/screenshots"
|
||||
)
|
||||
|
||||
url = manager.get_linkpeek_url("https://example.com/page")
|
||||
|
||||
assert "/screenshots/" in url
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_domains_empty(self):
|
||||
"""Test listing domains on empty vault."""
|
||||
manager = VaultManager(
|
||||
html_vault_base=self.html_base,
|
||||
media_vault_base=self.media_base,
|
||||
linkpeek_vault_base=self.linkpeek_base,
|
||||
)
|
||||
|
||||
# Create the base directory
|
||||
os.makedirs(self.html_base, exist_ok=True)
|
||||
|
||||
domains = await manager.list_domains("html")
|
||||
|
||||
assert domains == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_all_stats(self):
|
||||
"""Test getting all stats."""
|
||||
manager = VaultManager(
|
||||
html_vault_base=self.html_base,
|
||||
media_vault_base=self.media_base,
|
||||
linkpeek_vault_base=self.linkpeek_base,
|
||||
)
|
||||
|
||||
# Create base directories
|
||||
os.makedirs(self.html_base, exist_ok=True)
|
||||
os.makedirs(self.media_base, exist_ok=True)
|
||||
os.makedirs(self.linkpeek_base, exist_ok=True)
|
||||
|
||||
stats = await manager.get_all_stats()
|
||||
|
||||
assert 'html_vaults' in stats
|
||||
assert 'media_vaults' in stats
|
||||
assert 'linkpeek_vaults' in stats
|
||||
|
||||
|
||||
class TestVaultManagerDomainExtraction:
|
||||
"""Test domain extraction from URLs."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.manager = VaultManager(
|
||||
html_vault_base=os.path.join(self.temp_dir, "html"),
|
||||
media_vault_base=os.path.join(self.temp_dir, "media"),
|
||||
linkpeek_vault_base=os.path.join(self.temp_dir, "linkpeek"),
|
||||
)
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_extract_domain_basic(self):
|
||||
"""Test extracting domain from basic URL."""
|
||||
vault = self.manager.get_html_vault("example.com")
|
||||
assert vault.domain == "example.com"
|
||||
|
||||
def test_vaults_for_subdomain(self):
|
||||
"""Test getting vaults for subdomain."""
|
||||
html, media, linkpeek = self.manager.get_vaults_for_url(
|
||||
"https://sub.example.com/page"
|
||||
)
|
||||
|
||||
assert html.domain == "sub.example.com"
|
||||
assert media.domain == "sub.example.com"
|
||||
|
||||
def test_vaults_for_url_with_port(self):
|
||||
"""Test getting vaults for URL with port."""
|
||||
html, media, linkpeek = self.manager.get_vaults_for_url(
|
||||
"https://example.com:8080/page"
|
||||
)
|
||||
|
||||
# Domain includes port
|
||||
assert "example.com" in html.domain
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
299
tests/unit/test_screenshot.py
Normal file
299
tests/unit/test_screenshot.py
Normal file
|
|
@ -0,0 +1,299 @@
|
|||
"""
|
||||
Tests for screenshot module.
|
||||
|
||||
Tests async screenshot capture configuration and functionality.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, AsyncMock, patch, MagicMock
|
||||
import asyncio
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
from screenshot import ScreenshotConfig, ScreenshotCapture
|
||||
|
||||
|
||||
class TestScreenshotConfig:
|
||||
"""Test ScreenshotConfig dataclass."""
|
||||
|
||||
def test_default_config(self):
|
||||
"""Test default configuration values."""
|
||||
config = ScreenshotConfig()
|
||||
|
||||
assert config.enabled is False
|
||||
assert config.width == 1280
|
||||
assert config.height == 1024
|
||||
assert config.delay == 1000
|
||||
assert config.user_agent is None
|
||||
|
||||
def test_custom_config(self):
|
||||
"""Test custom configuration values."""
|
||||
config = ScreenshotConfig(
|
||||
enabled=True,
|
||||
width=1920,
|
||||
height=1080,
|
||||
delay=2000,
|
||||
user_agent="CustomBot/1.0",
|
||||
)
|
||||
|
||||
assert config.enabled is True
|
||||
assert config.width == 1920
|
||||
assert config.height == 1080
|
||||
assert config.delay == 2000
|
||||
assert config.user_agent == "CustomBot/1.0"
|
||||
|
||||
|
||||
class TestScreenshotCapture:
|
||||
"""Test ScreenshotCapture class."""
|
||||
|
||||
def test_capture_initialization_default(self):
|
||||
"""Test ScreenshotCapture with default config."""
|
||||
capture = ScreenshotCapture()
|
||||
|
||||
assert capture.config is not None
|
||||
assert capture.config.enabled is False
|
||||
assert capture._uri2png_available is None
|
||||
|
||||
def test_capture_initialization_custom(self):
|
||||
"""Test ScreenshotCapture with custom config."""
|
||||
config = ScreenshotConfig(enabled=True, width=800, height=600)
|
||||
capture = ScreenshotCapture(config=config)
|
||||
|
||||
assert capture.config.enabled is True
|
||||
assert capture.config.width == 800
|
||||
assert capture.config.height == 600
|
||||
|
||||
|
||||
class TestScreenshotCaptureAvailability:
|
||||
"""Test uri2png availability checking."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_available_caches_result(self):
|
||||
"""Test that availability check is cached."""
|
||||
capture = ScreenshotCapture()
|
||||
|
||||
with patch('asyncio.create_subprocess_exec', new_callable=AsyncMock) as mock_exec:
|
||||
mock_proc = AsyncMock()
|
||||
mock_proc.returncode = 0
|
||||
mock_proc.wait = AsyncMock(return_value=0)
|
||||
mock_exec.return_value = mock_proc
|
||||
|
||||
# First call
|
||||
result1 = await capture.is_available()
|
||||
# Second call should use cache
|
||||
result2 = await capture.is_available()
|
||||
|
||||
assert result1 == result2
|
||||
# Should only call subprocess once
|
||||
assert mock_exec.call_count == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_available_when_installed(self):
|
||||
"""Test availability when uri2png is installed."""
|
||||
capture = ScreenshotCapture()
|
||||
|
||||
with patch('asyncio.create_subprocess_exec', new_callable=AsyncMock) as mock_exec:
|
||||
mock_proc = AsyncMock()
|
||||
mock_proc.returncode = 0
|
||||
mock_proc.wait = AsyncMock(return_value=0)
|
||||
mock_exec.return_value = mock_proc
|
||||
|
||||
result = await capture.is_available()
|
||||
assert result is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_available_when_not_installed(self):
|
||||
"""Test availability when uri2png is not installed."""
|
||||
capture = ScreenshotCapture()
|
||||
|
||||
with patch('asyncio.create_subprocess_exec', new_callable=AsyncMock) as mock_exec:
|
||||
mock_proc = AsyncMock()
|
||||
mock_proc.returncode = 1
|
||||
mock_proc.wait = AsyncMock(return_value=1)
|
||||
mock_exec.return_value = mock_proc
|
||||
|
||||
result = await capture.is_available()
|
||||
assert result is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_available_handles_exception(self):
|
||||
"""Test availability handles exceptions gracefully."""
|
||||
capture = ScreenshotCapture()
|
||||
|
||||
with patch('asyncio.create_subprocess_exec', new_callable=AsyncMock) as mock_exec:
|
||||
mock_exec.side_effect = Exception("Process error")
|
||||
|
||||
result = await capture.is_available()
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestScreenshotCaptureMethod:
|
||||
"""Test the capture method."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capture_disabled_returns_none(self):
|
||||
"""Test capture returns None when disabled."""
|
||||
config = ScreenshotConfig(enabled=False)
|
||||
capture = ScreenshotCapture(config=config)
|
||||
|
||||
result = await capture.capture("https://example.com")
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capture_unavailable_returns_none(self):
|
||||
"""Test capture returns None when uri2png unavailable."""
|
||||
config = ScreenshotConfig(enabled=True)
|
||||
capture = ScreenshotCapture(config=config)
|
||||
|
||||
with patch.object(capture, 'is_available', new_callable=AsyncMock, return_value=False):
|
||||
result = await capture.capture("https://example.com")
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capture_success(self):
|
||||
"""Test successful screenshot capture."""
|
||||
config = ScreenshotConfig(enabled=True)
|
||||
capture = ScreenshotCapture(config=config)
|
||||
|
||||
# Create a temp PNG file to simulate screenshot
|
||||
with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as tmp:
|
||||
# Write some fake PNG data
|
||||
fake_png = b'\x89PNG\r\n\x1a\n' + b'fake png data'
|
||||
tmp.write(fake_png)
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
with patch.object(capture, 'is_available', new_callable=AsyncMock, return_value=True):
|
||||
with patch('asyncio.create_subprocess_exec', new_callable=AsyncMock) as mock_exec:
|
||||
mock_proc = AsyncMock()
|
||||
mock_proc.returncode = 0
|
||||
mock_proc.communicate = AsyncMock(return_value=(b'', b''))
|
||||
mock_exec.return_value = mock_proc
|
||||
|
||||
with patch('tempfile.NamedTemporaryFile') as mock_temp:
|
||||
mock_temp.return_value.__enter__ = Mock(return_value=Mock(name=tmp_path))
|
||||
mock_temp.return_value.__exit__ = Mock(return_value=False)
|
||||
|
||||
# Can't fully test without uri2png, but verify structure
|
||||
result = await capture.capture("https://example.com")
|
||||
# Result may be None if file checks fail, but no exception
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.unlink(tmp_path)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capture_timeout(self):
|
||||
"""Test capture handles timeout."""
|
||||
config = ScreenshotConfig(enabled=True)
|
||||
capture = ScreenshotCapture(config=config)
|
||||
|
||||
with patch.object(capture, 'is_available', new_callable=AsyncMock, return_value=True):
|
||||
with patch('asyncio.create_subprocess_exec', new_callable=AsyncMock) as mock_exec:
|
||||
mock_proc = AsyncMock()
|
||||
mock_proc.communicate = AsyncMock(side_effect=asyncio.TimeoutError())
|
||||
mock_proc.kill = Mock()
|
||||
mock_proc.wait = AsyncMock()
|
||||
mock_exec.return_value = mock_proc
|
||||
|
||||
with patch('tempfile.NamedTemporaryFile'):
|
||||
result = await capture.capture("https://slow-site.com")
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capture_process_failure(self):
|
||||
"""Test capture handles process failure."""
|
||||
config = ScreenshotConfig(enabled=True)
|
||||
capture = ScreenshotCapture(config=config)
|
||||
|
||||
with patch.object(capture, 'is_available', new_callable=AsyncMock, return_value=True):
|
||||
with patch('asyncio.create_subprocess_exec', new_callable=AsyncMock) as mock_exec:
|
||||
mock_proc = AsyncMock()
|
||||
mock_proc.returncode = 1
|
||||
mock_proc.communicate = AsyncMock(return_value=(b'', b'Error'))
|
||||
mock_exec.return_value = mock_proc
|
||||
|
||||
with patch('tempfile.NamedTemporaryFile'):
|
||||
result = await capture.capture("https://error-site.com")
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestScreenshotCaptureToFile:
|
||||
"""Test capture_to_file method."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capture_to_file_success(self):
|
||||
"""Test successful capture to file."""
|
||||
config = ScreenshotConfig(enabled=True)
|
||||
capture = ScreenshotCapture(config=config)
|
||||
|
||||
fake_result = {
|
||||
'data': b'fake png data',
|
||||
'md5_hash': 'abc123',
|
||||
'mime_type': 'image/png',
|
||||
'size': 13,
|
||||
}
|
||||
|
||||
with patch.object(capture, 'capture', new_callable=AsyncMock, return_value=fake_result):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output_path = os.path.join(tmpdir, 'screenshot.png')
|
||||
result = await capture.capture_to_file("https://example.com", output_path)
|
||||
|
||||
assert result is True
|
||||
assert os.path.exists(output_path)
|
||||
with open(output_path, 'rb') as f:
|
||||
assert f.read() == b'fake png data'
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capture_to_file_failure(self):
|
||||
"""Test capture_to_file when capture fails."""
|
||||
config = ScreenshotConfig(enabled=True)
|
||||
capture = ScreenshotCapture(config=config)
|
||||
|
||||
with patch.object(capture, 'capture', new_callable=AsyncMock, return_value=None):
|
||||
result = await capture.capture_to_file("https://example.com", "/tmp/test.png")
|
||||
assert result is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capture_to_file_write_error(self):
|
||||
"""Test capture_to_file handles write errors."""
|
||||
config = ScreenshotConfig(enabled=True)
|
||||
capture = ScreenshotCapture(config=config)
|
||||
|
||||
fake_result = {
|
||||
'data': b'fake png data',
|
||||
'md5_hash': 'abc123',
|
||||
'mime_type': 'image/png',
|
||||
'size': 13,
|
||||
}
|
||||
|
||||
with patch.object(capture, 'capture', new_callable=AsyncMock, return_value=fake_result):
|
||||
# Try to write to invalid path
|
||||
result = await capture.capture_to_file(
|
||||
"https://example.com",
|
||||
"/nonexistent/path/screenshot.png"
|
||||
)
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestScreenshotResultStructure:
|
||||
"""Test screenshot result structure."""
|
||||
|
||||
def test_expected_result_keys(self):
|
||||
"""Document expected keys in capture result."""
|
||||
expected_keys = ['data', 'md5_hash', 'mime_type', 'size', 'source_uri']
|
||||
|
||||
# Just document the structure - actual capture requires uri2png
|
||||
result = {
|
||||
'data': b'png bytes',
|
||||
'md5_hash': 'd41d8cd98f00b204e9800998ecf8427e',
|
||||
'mime_type': 'image/png',
|
||||
'size': 1234,
|
||||
'source_uri': 'https://example.com',
|
||||
}
|
||||
|
||||
for key in expected_keys:
|
||||
assert key in result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
330
tests/unit/test_storage.py
Normal file
330
tests/unit/test_storage.py
Normal file
|
|
@ -0,0 +1,330 @@
|
|||
"""
|
||||
Tests for storage module (ImageVault).
|
||||
|
||||
Tests async content-addressable storage with MD5 hashing.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import os
|
||||
import tempfile
|
||||
import shutil
|
||||
import hashlib
|
||||
|
||||
from storage import ImageVault
|
||||
|
||||
|
||||
class TestImageVaultBasics:
|
||||
"""Test basic ImageVault functionality."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.vault_path = os.path.join(self.temp_dir, "test_vault")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vault_initialization(self):
|
||||
"""Test ImageVault initializes correctly."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
assert vault._initialized is True
|
||||
assert os.path.exists(self.vault_path)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vault_creates_subdirectories(self):
|
||||
"""Test that vault creates hash bucket subdirectories."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
# Should create 256 subdirectories (00-ff)
|
||||
subdirs = os.listdir(self.vault_path)
|
||||
assert len(subdirs) == 256
|
||||
|
||||
# Check some specific ones
|
||||
assert "00" in subdirs
|
||||
assert "ff" in subdirs
|
||||
assert "a5" in subdirs
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vault_double_init(self):
|
||||
"""Test that double initialization is safe."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
await vault.init() # Should not raise
|
||||
|
||||
assert vault._initialized is True
|
||||
|
||||
|
||||
class TestImageVaultStorage:
|
||||
"""Test ImageVault storage operations."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.vault_path = os.path.join(self.temp_dir, "test_vault")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_store_and_retrieve(self):
|
||||
"""Test storing and retrieving data."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
# Create test data
|
||||
data = b"Hello, World! This is test image data."
|
||||
md5_hash = hashlib.md5(data).hexdigest()
|
||||
|
||||
# Store
|
||||
path = await vault.store(md5_hash, data, extension="jpg")
|
||||
assert path is not None
|
||||
|
||||
# Retrieve
|
||||
retrieved = await vault.get(md5_hash)
|
||||
assert retrieved == data
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_store_with_extension(self):
|
||||
"""Test storing with file extension."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
data = b"PNG image data here"
|
||||
md5_hash = hashlib.md5(data).hexdigest()
|
||||
|
||||
path = await vault.store(md5_hash, data, extension="png")
|
||||
assert path.endswith(".png")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_store_without_extension(self):
|
||||
"""Test storing without file extension."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
data = b"Binary data without extension"
|
||||
md5_hash = hashlib.md5(data).hexdigest()
|
||||
|
||||
path = await vault.store(md5_hash, data)
|
||||
# Should just be the hash
|
||||
assert md5_hash in path
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exists_true(self):
|
||||
"""Test exists returns True for stored data."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
data = b"Test data for exists check"
|
||||
md5_hash = hashlib.md5(data).hexdigest()
|
||||
|
||||
await vault.store(md5_hash, data)
|
||||
assert await vault.exists(md5_hash) is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exists_false(self):
|
||||
"""Test exists returns False for missing data."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
fake_hash = "0" * 32
|
||||
assert await vault.exists(fake_hash) is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_missing_returns_none(self):
|
||||
"""Test get returns None for missing data."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
fake_hash = "0" * 32
|
||||
result = await vault.get(fake_hash)
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestImageVaultDeduplication:
|
||||
"""Test ImageVault deduplication behavior."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.vault_path = os.path.join(self.temp_dir, "test_vault")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_data_same_path(self):
|
||||
"""Test that same data stored twice goes to same path."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
data = b"Duplicate test data"
|
||||
md5_hash = hashlib.md5(data).hexdigest()
|
||||
|
||||
path1 = await vault.store(md5_hash, data, extension="jpg")
|
||||
path2 = await vault.store(md5_hash, data, extension="jpg")
|
||||
|
||||
# Should be the same path
|
||||
assert path1 == path2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_different_data_different_path(self):
|
||||
"""Test that different data goes to different paths."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
data1 = b"First unique data"
|
||||
data2 = b"Second unique data"
|
||||
hash1 = hashlib.md5(data1).hexdigest()
|
||||
hash2 = hashlib.md5(data2).hexdigest()
|
||||
|
||||
path1 = await vault.store(hash1, data1)
|
||||
path2 = await vault.store(hash2, data2)
|
||||
|
||||
assert path1 != path2
|
||||
|
||||
|
||||
class TestImageVaultDeletion:
|
||||
"""Test ImageVault deletion operations."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.vault_path = os.path.join(self.temp_dir, "test_vault")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_existing(self):
|
||||
"""Test deleting existing file."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
data = b"Data to be deleted"
|
||||
md5_hash = hashlib.md5(data).hexdigest()
|
||||
|
||||
await vault.store(md5_hash, data)
|
||||
assert await vault.exists(md5_hash) is True
|
||||
|
||||
result = await vault.delete(md5_hash)
|
||||
assert result is True
|
||||
assert await vault.exists(md5_hash) is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_missing(self):
|
||||
"""Test deleting non-existent file."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
fake_hash = "0" * 32
|
||||
result = await vault.delete(fake_hash)
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestImageVaultPath:
|
||||
"""Test ImageVault path operations."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.vault_path = os.path.join(self.temp_dir, "test_vault")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_path_existing(self):
|
||||
"""Test getting path for existing file."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
data = b"Test data for path"
|
||||
md5_hash = hashlib.md5(data).hexdigest()
|
||||
|
||||
await vault.store(md5_hash, data, extension="bin")
|
||||
path = await vault.get_path(md5_hash)
|
||||
|
||||
assert path is not None
|
||||
assert path.exists()
|
||||
assert md5_hash in str(path)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_path_missing(self):
|
||||
"""Test getting path for missing file."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
fake_hash = "0" * 32
|
||||
path = await vault.get_path(fake_hash)
|
||||
assert path is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_path_uses_hash_prefix(self):
|
||||
"""Test that path uses first 2 chars as subdirectory."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
data = b"Test data for subdir check"
|
||||
md5_hash = hashlib.md5(data).hexdigest()
|
||||
prefix = md5_hash[:2]
|
||||
|
||||
path = await vault.store(md5_hash, data)
|
||||
|
||||
# Path should contain the prefix subdirectory
|
||||
assert f"/{prefix}/" in path or f"\\{prefix}\\" in path
|
||||
|
||||
|
||||
class TestImageVaultStats:
|
||||
"""Test ImageVault statistics."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.vault_path = os.path.join(self.temp_dir, "test_vault")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stats_empty_vault(self):
|
||||
"""Test stats on empty vault."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
stats = await vault.stats()
|
||||
assert stats['count'] == 0
|
||||
assert stats['total_size_bytes'] == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stats_with_files(self):
|
||||
"""Test stats with files in vault."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
# Store some files
|
||||
for i in range(3):
|
||||
data = f"Test data {i}".encode()
|
||||
md5_hash = hashlib.md5(data).hexdigest()
|
||||
await vault.store(md5_hash, data)
|
||||
|
||||
stats = await vault.stats()
|
||||
assert stats['count'] == 3
|
||||
assert stats['total_size_bytes'] > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stats_backend_type(self):
|
||||
"""Test stats reports correct backend."""
|
||||
vault = ImageVault(vault_path=self.vault_path)
|
||||
await vault.init()
|
||||
|
||||
stats = await vault.stats()
|
||||
# Should be 'directory' when filevault is not available
|
||||
assert 'backend' in stats
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
Loading…
Add table
Add a link
Reference in a new issue