diff --git a/make_post_sell/lib/checksums.py b/make_post_sell/lib/checksums.py new file mode 100644 index 0000000..247320f --- /dev/null +++ b/make_post_sell/lib/checksums.py @@ -0,0 +1,90 @@ +"""checksums.py — background MD5 + SHA-256 computation for uploaded files. + +After a file lands in its final S3 location, compute both checksums by streaming +the object through Python's hashlib. Results are stored in the product's +file_metadata under {"checksums": {file_key: {"md5": "...", "sha256": "..."}}}. + +Fire-and-forget daemon thread — same pattern as torrent.py and s3_mirror.py. +""" + +import hashlib +import logging +import threading + +import boto3 + +log = logging.getLogger(__name__) + + +def _capture_s3_creds(client): + creds = client._request_signer._credentials + meta = client.meta + return { + "endpoint": meta.endpoint_url, + "region": meta.region_name, + "access_key": creds.access_key, + "secret_key": creds.secret_key, + } + + +def _make_client(creds): + return boto3.session.Session().client( + "s3", + region_name=creds["region"], + endpoint_url=creds["endpoint"], + aws_access_key_id=creds["access_key"], + aws_secret_access_key=creds["secret_key"], + ) + + +def compute_checksums(s3_client, bucket, s3_key, file_key, product_id, session_factory): + """Stream s3_key, compute MD5 + SHA-256, save to product file_metadata. + + Designed to run in a background thread. + """ + log.info("checksums: computing for product=%s key=%s", product_id, s3_key) + + md5 = hashlib.md5() + sha256 = hashlib.sha256() + + try: + response = s3_client.get_object(Bucket=bucket, Key=s3_key) + for chunk in response["Body"].iter_chunks(chunk_size=1024 * 1024): + md5.update(chunk) + sha256.update(chunk) + except Exception: + log.exception("checksums: failed to stream s3://%s/%s", bucket, s3_key) + return + + md5_hex = md5.hexdigest() + sha256_hex = sha256.hexdigest() + log.info("checksums: product=%s key=%s md5=%s sha256=%s", + product_id, s3_key, md5_hex, sha256_hex[:16] + "...") + + try: + with session_factory() as session: + from ..models.product import Product + product = session.get(Product, product_id) + if product is not None: + product.set_checksum(file_key, md5_hex, sha256_hex) + session.add(product) + session.commit() + log.info("checksums: saved for product=%s file_key=%s", product_id, file_key) + except Exception: + log.exception("checksums: failed to save for product=%s", product_id) + + +def compute_checksums_async(s3_client, bucket, s3_key, file_key, product_id, session_factory): + """Fire-and-forget: compute checksums in a daemon thread.""" + creds = _capture_s3_creds(s3_client) + + def _run(): + try: + fresh_client = _make_client(creds) + compute_checksums(fresh_client, bucket, s3_key, file_key, product_id, session_factory) + except Exception: + log.exception("checksums: background thread failed for product=%s", product_id) + + t = threading.Thread(target=_run, daemon=True, name=f"checksums-{product_id}-{file_key}") + t.start() + log.info("checksums: background thread started for product=%s file_key=%s", product_id, file_key) diff --git a/make_post_sell/models/product.py b/make_post_sell/models/product.py index 99975b6..2f126e4 100644 --- a/make_post_sell/models/product.py +++ b/make_post_sell/models/product.py @@ -385,6 +385,18 @@ class Product(RBase, Base): def originals(self): return self.file_metadata.get("originals", {}) + @property + def checksums(self): + return self.file_metadata.get("checksums", {}) + + def set_checksum(self, file_key, md5_hex, sha256_hex): + tmp = self.file_metadata + if "checksums" not in tmp: + tmp["checksums"] = {} + tmp["checksums"][file_key] = {"md5": md5_hex, "sha256": sha256_hex} + self._file_metadata = tmp + self.json_file_metadata = json.dumps(tmp) + def set_file_metadata(self, file_key, extension, original_filename): tmp = self.file_metadata tmp["extensions"][file_key] = extension diff --git a/make_post_sell/routes.py b/make_post_sell/routes.py index 5a1d765..1b8da08 100644 --- a/make_post_sell/routes.py +++ b/make_post_sell/routes.py @@ -230,3 +230,7 @@ def includeme(config): # Torrent backfill progress config.add_route("shop_torrent_backfill_status", "/s/{shop_id}/torrent-backfill-status") + + # Checksum backfill + config.add_route("shop_checksum_backfill", "/s/{shop_id}/checksum-backfill") + config.add_route("shop_checksum_backfill_status", "/s/{shop_id}/checksum-backfill-status") diff --git a/make_post_sell/templates/content.j2 b/make_post_sell/templates/content.j2 index db46b6c..e6aee94 100644 --- a/make_post_sell/templates/content.j2 +++ b/make_post_sell/templates/content.j2 @@ -138,6 +138,17 @@ Created {{ product.human_created_timestamp }}{% if product.human_created_timestamp != product.human_updated_timestamp %} · Updated {{ product.human_updated_timestamp }}{% endif %}{% if product.human_view_count %} · {{ product.human_view_count }}{% endif %} {% endif %} + + {% set cs = product.checksums.get("product", {}) %} + {% if cs.get("sha256") or cs.get("md5") %} +
+ Checksums + + {% if cs.get("sha256") %}{% endif %} + {% if cs.get("md5") %}{% endif %} +
SHA-256{{ cs["sha256"] }}
MD5{{ cs["md5"] }}
+
+ {% endif %}
diff --git a/make_post_sell/templates/shop_settings.j2 b/make_post_sell/templates/shop_settings.j2 index c4c0186..e82a001 100644 --- a/make_post_sell/templates/shop_settings.j2 +++ b/make_post_sell/templates/shop_settings.j2 @@ -1321,6 +1321,21 @@ Existing sales honored for download buy purchasers. +
+

File Checksums

+

MD5 and SHA-256 checksums are computed for every uploaded file and displayed on content pages so visitors can verify integrity.

+
+ +
+
+

Backfill progress

+
+
+
+

Loading…

+
+
+ @@ -1383,6 +1398,35 @@ function toggleCryptoWallets() { poll(); timer = setInterval(poll, 4000); })(); + +(function() { + const bar = document.getElementById('checksum-bar'); + const status = document.getElementById('checksum-status'); + if (!bar || !status) return; + + let timer; + + function poll() { + fetch('/s/{{ request.shop.id }}/checksum-backfill-status') + .then(function(r) { return r.json(); }) + .then(function(d) { + const pct = d.total > 0 ? Math.round(d.done / d.total * 100) : 100; + bar.style.width = pct + '%'; + if (d.total === 0) { + status.textContent = 'No files uploaded yet.'; + } else if (d.done === d.total) { + status.textContent = 'Done — ' + d.done + ' of ' + d.total + ' files have checksums.'; + clearInterval(timer); + } else { + status.textContent = d.done + ' / ' + d.total + ' done (' + d.remaining + ' pending)…'; + } + }) + .catch(function() { status.textContent = 'Could not load status.'; }); + } + + poll(); + timer = setInterval(poll, 4000); +})(); diff --git a/make_post_sell/views/product.py b/make_post_sell/views/product.py index 29cc4ce..61ee26d 100644 --- a/make_post_sell/views/product.py +++ b/make_post_sell/views/product.py @@ -439,6 +439,17 @@ def product_edit(request): cdn_endpoint=request.shop_cdn_endpoint, ) + # Compute MD5 + SHA-256 for every uploaded file. + from ..lib.checksums import compute_checksums_async + compute_checksums_async( + request.shop_uploads_client, + request.shop_bucket_name, + s3_key=f"{product.s3_path}/{file_key}", + file_key=file_key, + product_id=product.id, + session_factory=request.registry["dbsession_factory"], + ) + # Generate vocal isolation tracks if this is audio/video from ..models.product import get_media_type from ..lib.karaoke import process_karaoke diff --git a/make_post_sell/views/shop.py b/make_post_sell/views/shop.py index 73ecaff..ebeed38 100644 --- a/make_post_sell/views/shop.py +++ b/make_post_sell/views/shop.py @@ -118,6 +118,48 @@ def _torrent_backfill_async(request, shop): ) +def _checksum_backfill_async(request, shop): + """Fire-and-forget: compute checksums for all product files in a shop + that do not yet have a checksum recorded. Safe to re-run — skips files + that already have both md5 and sha256.""" + from ..lib.checksums import compute_checksums_async + from ..models.product import Product + + all_file_keys = ["product", "preview", "thumbnail1", "thumbnail2", "thumbnail3", "thumbnail4"] + + products = ( + request.dbsession.query(Product) + .filter(Product.shop_id == shop.id) + .all() + ) + bucket = request.shop_bucket_name + s3_client = request.shop_uploads_client + session_factory = request.registry["dbsession_factory"] + + queued = 0 + for product in products: + for file_key in all_file_keys: + if file_key not in product.originals: + continue # file not uploaded + existing = product.checksums.get(file_key, {}) + if existing.get("md5") and existing.get("sha256"): + continue # already computed + compute_checksums_async( + s3_client, + bucket, + s3_key=f"{product.s3_path}/{file_key}", + file_key=file_key, + product_id=product.id, + session_factory=session_factory, + ) + queued += 1 + + import logging + logging.getLogger(__name__).info( + "checksum backfill: queued %d file(s) for shop=%s", queued, shop.id + ) + + def get_shop_from_matchdict(request, prefetched_shop=None): """ This function uses the shop_id from the url path @@ -1553,3 +1595,54 @@ def torrent_backfill_status(request): "done": done_count, "remaining": total - done_count, } + + +# ── Checksum backfill ───────────────────────────────────────────────────────── + +_CHECKSUM_FILE_KEYS = ["product", "preview", "thumbnail1", "thumbnail2", "thumbnail3", "thumbnail4"] + + +@view_config(route_name="shop_checksum_backfill", renderer="json", request_method="POST") +@user_required() +@shop_owner_required() +def checksum_backfill(request): + """Trigger background checksum computation for all files that don't have one yet.""" + _checksum_backfill_async(request, request.shop) + request.session.flash(("Checksum backfill started.", "success")) + return HTTPFound(request.route_url("shop_settings", shop_id=request.shop.id)) + + +@view_config(route_name="shop_checksum_backfill_status", renderer="json", request_method="GET") +@user_required() +@shop_owner_required() +def checksum_backfill_status(request): + """Return checksum backfill progress. + + JSON: { total, done, remaining } + - total: count of (product, file_key) pairs that have a file + - done: of those, how many have both md5 + sha256 + """ + from ..models.product import Product + + products = ( + request.dbsession.query(Product) + .filter(Product.shop_id == request.shop.id) + .all() + ) + + total = 0 + done_count = 0 + for p in products: + for file_key in _CHECKSUM_FILE_KEYS: + if file_key not in p.originals: + continue + total += 1 + cs = p.checksums.get(file_key, {}) + if cs.get("md5") and cs.get("sha256"): + done_count += 1 + + return { + "total": total, + "done": done_count, + "remaining": total - done_count, + }