engineering/pig.py
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix UNIQUE constraint error when pages have identical screenshots

Browse source
This commit is contained in:
Russell Ballestrini 2025-12-30 16:20:15 -05:00
parent 3fa3c9722b
commit 7daae6c0dc
1 changed files with 19 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

32
neopig.py
View file

@ -1532,20 +1532,26 @@ async def backfill_screenshots(
old_media = old_media_result.scalar_one_or_none()
if old_media and old_hash != new_hash:
new_media = Media(
md5_hash=new_hash,
media_type='screenshot',
mime_type=new_mime,
file_size=new_size,
keywords=old_media.keywords,
alt_text=old_media.alt_text,
title=old_media.title,
first_seen_at=datetime.now(timezone.utc).isoformat(),
analysis_status=old_media.analysis_status,
analysis_result=old_media.analysis_result,
# Check if new hash already exists (duplicate screenshot from another page)
existing = await local_session.execute(
select(Media).where(Media.md5_hash == new_hash)
)
local_session.add(new_media)
await local_session.flush()
if not existing.scalar_one_or_none():
new_media = Media(
md5_hash=new_hash,
media_type='screenshot',
mime_type=new_mime,
file_size=new_size,
keywords=old_media.keywords,
alt_text=old_media.alt_text,
title=old_media.title,
first_seen_at=datetime.now(timezone.utc).isoformat(),
analysis_status=old_media.analysis_status,
analysis_result=old_media.analysis_result,
)
local_session.add(new_media)
await local_session.flush()
# Update sources to point to new hash and delete old media
await local_session.execute(
update(MediaSource).where(MediaSource.md5_hash == old_hash).values(md5_hash=new_hash)
)