checksums: compute MD5+SHA-256 for every uploaded file, display on content page

- lib/checksums.py: background thread streams from S3, computes both digests
- models/product.py: checksums property + set_checksum stored in file_metadata["checksums"]
- views/product.py: trigger compute_checksums_async after every file upload
- views/shop.py: checksum backfill function (skips files already computed),
  checksum_backfill POST view, checksum_backfill_status JSON view
- routes.py: /s/{shop_id}/checksum-backfill + /s/{shop_id}/checksum-backfill-status
- templates/content.j2: <details> block near date metadata — SHA-256 then MD5
- templates/shop_settings.j2: Backfill Checksums button + progress bar with 4s polling

make_post_sell/lib/checksums.py

@@ -0,0 +1,90 @@
+"""checksums.py — background MD5 + SHA-256 computation for uploaded files.
+
+After a file lands in its final S3 location, compute both checksums by streaming
+the object through Python's hashlib.  Results are stored in the product's
+file_metadata under {"checksums": {file_key: {"md5": "...", "sha256": "..."}}}.
+
+Fire-and-forget daemon thread — same pattern as torrent.py and s3_mirror.py.
+"""
+
+import hashlib
+import logging
+import threading
+
+import boto3
+
+log = logging.getLogger(__name__)
+
+
+def _capture_s3_creds(client):
+    creds = client._request_signer._credentials
+    meta = client.meta
+    return {
+        "endpoint": meta.endpoint_url,
+        "region": meta.region_name,
+        "access_key": creds.access_key,
+        "secret_key": creds.secret_key,
+    }
+
+
+def _make_client(creds):
+    return boto3.session.Session().client(
+        "s3",
+        region_name=creds["region"],
+        endpoint_url=creds["endpoint"],
+        aws_access_key_id=creds["access_key"],
+        aws_secret_access_key=creds["secret_key"],
+    )
+
+
+def compute_checksums(s3_client, bucket, s3_key, file_key, product_id, session_factory):
+    """Stream s3_key, compute MD5 + SHA-256, save to product file_metadata.
+
+    Designed to run in a background thread.
+    """
+    log.info("checksums: computing for product=%s key=%s", product_id, s3_key)
+
+    md5 = hashlib.md5()
+    sha256 = hashlib.sha256()
+
+    try:
+        response = s3_client.get_object(Bucket=bucket, Key=s3_key)
+        for chunk in response["Body"].iter_chunks(chunk_size=1024 * 1024):
+            md5.update(chunk)
+            sha256.update(chunk)
+    except Exception:
+        log.exception("checksums: failed to stream s3://%s/%s", bucket, s3_key)
+        return
+
+    md5_hex = md5.hexdigest()
+    sha256_hex = sha256.hexdigest()
+    log.info("checksums: product=%s key=%s md5=%s sha256=%s",
+             product_id, s3_key, md5_hex, sha256_hex[:16] + "...")
+
+    try:
+        with session_factory() as session:
+            from ..models.product import Product
+            product = session.get(Product, product_id)
+            if product is not None:
+                product.set_checksum(file_key, md5_hex, sha256_hex)
+                session.add(product)
+                session.commit()
+                log.info("checksums: saved for product=%s file_key=%s", product_id, file_key)
+    except Exception:
+        log.exception("checksums: failed to save for product=%s", product_id)
+
+
+def compute_checksums_async(s3_client, bucket, s3_key, file_key, product_id, session_factory):
+    """Fire-and-forget: compute checksums in a daemon thread."""
+    creds = _capture_s3_creds(s3_client)
+
+    def _run():
+        try:
+            fresh_client = _make_client(creds)
+            compute_checksums(fresh_client, bucket, s3_key, file_key, product_id, session_factory)
+        except Exception:
+            log.exception("checksums: background thread failed for product=%s", product_id)
+
+    t = threading.Thread(target=_run, daemon=True, name=f"checksums-{product_id}-{file_key}")
+    t.start()
+    log.info("checksums: background thread started for product=%s file_key=%s", product_id, file_key)

make_post_sell/models/product.py

@@ -385,6 +385,18 @@ class Product(RBase, Base):
     def originals(self):
         return self.file_metadata.get("originals", {})
 
+    @property
+    def checksums(self):
+        return self.file_metadata.get("checksums", {})
+
+    def set_checksum(self, file_key, md5_hex, sha256_hex):
+        tmp = self.file_metadata
+        if "checksums" not in tmp:
+            tmp["checksums"] = {}
+        tmp["checksums"][file_key] = {"md5": md5_hex, "sha256": sha256_hex}
+        self._file_metadata = tmp
+        self.json_file_metadata = json.dumps(tmp)
+
     def set_file_metadata(self, file_key, extension, original_filename):
         tmp = self.file_metadata
         tmp["extensions"][file_key] = extension

make_post_sell/routes.py

@@ -230,3 +230,7 @@ def includeme(config):
 
     # Torrent backfill progress
     config.add_route("shop_torrent_backfill_status",   "/s/{shop_id}/torrent-backfill-status")
+
+    # Checksum backfill
+    config.add_route("shop_checksum_backfill",         "/s/{shop_id}/checksum-backfill")
+    config.add_route("shop_checksum_backfill_status",  "/s/{shop_id}/checksum-backfill-status")

make_post_sell/templates/content.j2

@@ -138,6 +138,17 @@
       Created {{ product.human_created_timestamp }}{% if product.human_created_timestamp != product.human_updated_timestamp %} · Updated {{ product.human_updated_timestamp }}{% endif %}{% if product.human_view_count %} · <span class="product-view-count">{{ product.human_view_count }}</span>{% endif %}
     </div>
     {% endif %}
+
+    {% set cs = product.checksums.get("product", {}) %}
+    {% if cs.get("sha256") or cs.get("md5") %}
+    <details class="product-checksums">
+      <summary>Checksums</summary>
+      <table class="checksum-table">
+        {% if cs.get("sha256") %}<tr><td>SHA-256</td><td><code>{{ cs["sha256"] }}</code></td></tr>{% endif %}
+        {% if cs.get("md5") %}<tr><td>MD5</td><td><code>{{ cs["md5"] }}</code></td></tr>{% endif %}
+      </table>
+    </details>
+    {% endif %}
   </div>
 
   <div class="product-comments">

make_post_sell/templates/shop_settings.j2

@@ -1321,6 +1321,21 @@ Existing sales honored for download buy purchasers.
 
   </section>
 
+  <section class="shop-settings-checksum well2" style="margin-top: var(--space-6);">
+    <h3>File Checksums</h3>
+    <p>MD5 and SHA-256 checksums are computed for every uploaded file and displayed on content pages so visitors can verify integrity.</p>
+    <form method="post" action="/s/{{ request.shop.id }}/checksum-backfill" onsubmit="submit.disabled = true; return true;">
+      <input type="submit" name="submit" class="mps-submit" value="Backfill Checksums" />
+    </form>
+    <div id="checksum-backfill" style="margin-top: var(--space-4);">
+      <p style="margin: 0 0 var(--space-2);">Backfill progress</p>
+      <div style="background: var(--color-surface-alt); border-radius: var(--radius-md); height: 12px; overflow: hidden;">
+        <div id="checksum-bar" style="height: 100%; width: 0%; background: var(--color-primary, #16a34a); transition: width 0.4s ease;"></div>
+      </div>
+      <p id="checksum-status" style="margin: var(--space-2) 0 0; font-size: 0.85em; color: var(--color-text-muted);">Loading…</p>
+    </div>
+  </section>
+
 </section>
 
 
@@ -1383,6 +1398,35 @@ function toggleCryptoWallets() {
     poll();
     timer = setInterval(poll, 4000);
 })();
+
+(function() {
+    const bar    = document.getElementById('checksum-bar');
+    const status = document.getElementById('checksum-status');
+    if (!bar || !status) return;
+
+    let timer;
+
+    function poll() {
+        fetch('/s/{{ request.shop.id }}/checksum-backfill-status')
+            .then(function(r) { return r.json(); })
+            .then(function(d) {
+                const pct = d.total > 0 ? Math.round(d.done / d.total * 100) : 100;
+                bar.style.width = pct + '%';
+                if (d.total === 0) {
+                    status.textContent = 'No files uploaded yet.';
+                } else if (d.done === d.total) {
+                    status.textContent = 'Done — ' + d.done + ' of ' + d.total + ' files have checksums.';
+                    clearInterval(timer);
+                } else {
+                    status.textContent = d.done + ' / ' + d.total + ' done (' + d.remaining + ' pending)…';
+                }
+            })
+            .catch(function() { status.textContent = 'Could not load status.'; });
+    }
+
+    poll();
+    timer = setInterval(poll, 4000);
+})();
 </script>
 <script src="/static/js/shop-settings.js"></script>
 

make_post_sell/views/product.py

@@ -439,6 +439,17 @@ def product_edit(request):
                     cdn_endpoint=request.shop_cdn_endpoint,
                 )
 
+        # Compute MD5 + SHA-256 for every uploaded file.
+        from ..lib.checksums import compute_checksums_async
+        compute_checksums_async(
+            request.shop_uploads_client,
+            request.shop_bucket_name,
+            s3_key=f"{product.s3_path}/{file_key}",
+            file_key=file_key,
+            product_id=product.id,
+            session_factory=request.registry["dbsession_factory"],
+        )
+
         # Generate vocal isolation tracks if this is audio/video
         from ..models.product import get_media_type
         from ..lib.karaoke import process_karaoke

make_post_sell/views/shop.py

@@ -118,6 +118,48 @@ def _torrent_backfill_async(request, shop):
     )
 
 
+def _checksum_backfill_async(request, shop):
+    """Fire-and-forget: compute checksums for all product files in a shop
+    that do not yet have a checksum recorded.  Safe to re-run — skips files
+    that already have both md5 and sha256."""
+    from ..lib.checksums import compute_checksums_async
+    from ..models.product import Product
+
+    all_file_keys = ["product", "preview", "thumbnail1", "thumbnail2", "thumbnail3", "thumbnail4"]
+
+    products = (
+        request.dbsession.query(Product)
+        .filter(Product.shop_id == shop.id)
+        .all()
+    )
+    bucket = request.shop_bucket_name
+    s3_client = request.shop_uploads_client
+    session_factory = request.registry["dbsession_factory"]
+
+    queued = 0
+    for product in products:
+        for file_key in all_file_keys:
+            if file_key not in product.originals:
+                continue  # file not uploaded
+            existing = product.checksums.get(file_key, {})
+            if existing.get("md5") and existing.get("sha256"):
+                continue  # already computed
+            compute_checksums_async(
+                s3_client,
+                bucket,
+                s3_key=f"{product.s3_path}/{file_key}",
+                file_key=file_key,
+                product_id=product.id,
+                session_factory=session_factory,
+            )
+            queued += 1
+
+    import logging
+    logging.getLogger(__name__).info(
+        "checksum backfill: queued %d file(s) for shop=%s", queued, shop.id
+    )
+
+
 def get_shop_from_matchdict(request, prefetched_shop=None):
     """
     This function uses the shop_id from the url path
@@ -1553,3 +1595,54 @@ def torrent_backfill_status(request):
         "done": done_count,
         "remaining": total - done_count,
     }
+
+
+# ── Checksum backfill ─────────────────────────────────────────────────────────
+
+_CHECKSUM_FILE_KEYS = ["product", "preview", "thumbnail1", "thumbnail2", "thumbnail3", "thumbnail4"]
+
+
+@view_config(route_name="shop_checksum_backfill", renderer="json", request_method="POST")
+@user_required()
+@shop_owner_required()
+def checksum_backfill(request):
+    """Trigger background checksum computation for all files that don't have one yet."""
+    _checksum_backfill_async(request, request.shop)
+    request.session.flash(("Checksum backfill started.", "success"))
+    return HTTPFound(request.route_url("shop_settings", shop_id=request.shop.id))
+
+
+@view_config(route_name="shop_checksum_backfill_status", renderer="json", request_method="GET")
+@user_required()
+@shop_owner_required()
+def checksum_backfill_status(request):
+    """Return checksum backfill progress.
+
+    JSON: { total, done, remaining }
+    - total: count of (product, file_key) pairs that have a file
+    - done:  of those, how many have both md5 + sha256
+    """
+    from ..models.product import Product
+
+    products = (
+        request.dbsession.query(Product)
+        .filter(Product.shop_id == request.shop.id)
+        .all()
+    )
+
+    total = 0
+    done_count = 0
+    for p in products:
+        for file_key in _CHECKSUM_FILE_KEYS:
+            if file_key not in p.originals:
+                continue
+            total += 1
+            cs = p.checksums.get(file_key, {})
+            if cs.get("md5") and cs.get("sha256"):
+                done_count += 1
+
+    return {
+        "total": total,
+        "done": done_count,
+        "remaining": total - done_count,
+    }