feat: add per-shop S3 mirror bucket for automatic upload sync
Shop owners can configure their own S3-compatible bucket (DigitalOcean Spaces, AWS S3, MinIO, Backblaze B2, etc.) to automatically mirror all uploaded files. The MPS main bucket remains the origin/CDN — the shop bucket is a fire-and-forget backup. Sync runs in daemon threads after each copy_object, capturing credentials as plain strings for thread safety with fresh boto3 clients per thread. Backfill button copies all existing files via double-fork process.
This commit is contained in:
parent
d5bf831bcd
commit
0579c87c43
7 changed files with 702 additions and 0 deletions
306
make_post_sell/lib/s3_mirror.py
Normal file
306
make_post_sell/lib/s3_mirror.py
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
"""S3 mirror — fire-and-forget sync of uploaded files to a shop's custom bucket.
|
||||
|
||||
When a shop has mirror_s3_* credentials configured and enabled, every file
|
||||
written to the MPS main bucket is copied to the shop's bucket in a background
|
||||
thread. The MPS bucket remains the origin/CDN — the shop bucket is a mirror.
|
||||
"""
|
||||
|
||||
import fcntl
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
|
||||
import boto3
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _make_mirror_client(endpoint, region, access_key, secret_key):
|
||||
"""Create a boto3 S3 client from mirror credentials."""
|
||||
session = boto3.session.Session()
|
||||
return session.client(
|
||||
"s3",
|
||||
region_name=region or "us-east-1",
|
||||
endpoint_url=endpoint,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
)
|
||||
|
||||
|
||||
def test_mirror_connection(shop):
|
||||
"""Validate shop mirror credentials by listing the bucket.
|
||||
|
||||
Returns (True, "") on success or (False, "error message") on failure.
|
||||
"""
|
||||
try:
|
||||
client = _make_mirror_client(
|
||||
shop.mirror_s3_endpoint,
|
||||
shop.mirror_s3_region,
|
||||
shop.mirror_s3_access_key,
|
||||
shop.mirror_s3_secret_key,
|
||||
)
|
||||
client.list_objects_v2(Bucket=shop.mirror_s3_bucket, MaxKeys=0)
|
||||
return True, ""
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def mirror_key(src_client, src_bucket, key, dst_client, dst_bucket,
|
||||
content_type=None, cache_control=None):
|
||||
"""Copy a single key from source bucket to mirror bucket.
|
||||
|
||||
Streams via get_object → put_object. The StreamingBody from get_object
|
||||
is passed directly as Body to put_object.
|
||||
"""
|
||||
try:
|
||||
resp = src_client.get_object(Bucket=src_bucket, Key=key)
|
||||
body = resp["Body"]
|
||||
|
||||
put_kwargs = {
|
||||
"Bucket": dst_bucket,
|
||||
"Key": key,
|
||||
"Body": body,
|
||||
}
|
||||
if content_type:
|
||||
put_kwargs["ContentType"] = content_type
|
||||
if cache_control:
|
||||
put_kwargs["CacheControl"] = cache_control
|
||||
|
||||
dst_client.put_object(**put_kwargs)
|
||||
log.info("Mirrored %s to %s/%s", key, dst_bucket, key)
|
||||
except Exception:
|
||||
log.exception("Failed to mirror key %s to bucket %s", key, dst_bucket)
|
||||
|
||||
|
||||
def _capture_shop_mirror_creds(shop):
|
||||
"""Capture mirror credentials as plain strings for thread safety.
|
||||
|
||||
The shop ORM object must not be accessed from a background thread
|
||||
after the request's DB session is closed.
|
||||
"""
|
||||
return {
|
||||
"endpoint": shop.mirror_s3_endpoint,
|
||||
"region": shop.mirror_s3_region,
|
||||
"bucket": shop.mirror_s3_bucket,
|
||||
"access_key": shop.mirror_s3_access_key,
|
||||
"secret_key": shop.mirror_s3_secret_key,
|
||||
}
|
||||
|
||||
|
||||
def _capture_source_creds(src_client):
|
||||
"""Capture source S3 client credentials for thread safety.
|
||||
|
||||
boto3 clients are not thread-safe — the background thread must
|
||||
create its own client.
|
||||
"""
|
||||
return {
|
||||
"endpoint": src_client._endpoint.host,
|
||||
"region": src_client.meta.region_name,
|
||||
"access_key": src_client._request_signer._credentials.access_key,
|
||||
"secret_key": src_client._request_signer._credentials.secret_key,
|
||||
}
|
||||
|
||||
|
||||
def mirror_key_async(src_client, src_bucket, key, shop,
|
||||
content_type=None, cache_control=None):
|
||||
"""Fire-and-forget: mirror a key to the shop's bucket in a daemon thread.
|
||||
|
||||
Does nothing if the shop has no mirror configured.
|
||||
"""
|
||||
if not shop.has_s3_mirror:
|
||||
return
|
||||
|
||||
mirror_creds = _capture_shop_mirror_creds(shop)
|
||||
source_creds = _capture_source_creds(src_client)
|
||||
|
||||
def _sync():
|
||||
try:
|
||||
sess = boto3.session.Session()
|
||||
src = sess.client(
|
||||
"s3",
|
||||
region_name=source_creds["region"],
|
||||
endpoint_url=source_creds["endpoint"],
|
||||
aws_access_key_id=source_creds["access_key"],
|
||||
aws_secret_access_key=source_creds["secret_key"],
|
||||
)
|
||||
dst = _make_mirror_client(
|
||||
mirror_creds["endpoint"],
|
||||
mirror_creds["region"],
|
||||
mirror_creds["access_key"],
|
||||
mirror_creds["secret_key"],
|
||||
)
|
||||
mirror_key(src, src_bucket, key, dst, mirror_creds["bucket"],
|
||||
content_type=content_type, cache_control=cache_control)
|
||||
except Exception:
|
||||
log.exception("Mirror thread failed for key %s", key)
|
||||
|
||||
t = threading.Thread(target=_sync, daemon=True)
|
||||
t.start()
|
||||
|
||||
|
||||
def mirror_keys_async(src_client, src_bucket, keys, shop,
|
||||
content_type=None, cache_control=None):
|
||||
"""Fire-and-forget: mirror multiple keys in a single daemon thread."""
|
||||
if not shop.has_s3_mirror:
|
||||
return
|
||||
|
||||
mirror_creds = _capture_shop_mirror_creds(shop)
|
||||
source_creds = _capture_source_creds(src_client)
|
||||
|
||||
def _sync():
|
||||
try:
|
||||
sess = boto3.session.Session()
|
||||
src = sess.client(
|
||||
"s3",
|
||||
region_name=source_creds["region"],
|
||||
endpoint_url=source_creds["endpoint"],
|
||||
aws_access_key_id=source_creds["access_key"],
|
||||
aws_secret_access_key=source_creds["secret_key"],
|
||||
)
|
||||
dst = _make_mirror_client(
|
||||
mirror_creds["endpoint"],
|
||||
mirror_creds["region"],
|
||||
mirror_creds["access_key"],
|
||||
mirror_creds["secret_key"],
|
||||
)
|
||||
for key in keys:
|
||||
mirror_key(src, src_bucket, key, dst, mirror_creds["bucket"],
|
||||
content_type=content_type, cache_control=cache_control)
|
||||
except Exception:
|
||||
log.exception("Mirror thread failed for keys %s", keys)
|
||||
|
||||
t = threading.Thread(target=_sync, daemon=True)
|
||||
t.start()
|
||||
|
||||
|
||||
def backfill_mirror_async(shop_id, session_factory, app_settings):
|
||||
"""Copy all existing S3 files for a shop to the mirror bucket.
|
||||
|
||||
Forks a child process so the backfill survives uWSGI worker recycling.
|
||||
Uses fcntl.flock on a lockfile for one-per-shop guard.
|
||||
"""
|
||||
shop_id_str = str(shop_id)
|
||||
lockfile = f"/tmp/s3_mirror_backfill_{shop_id_str}.lock"
|
||||
|
||||
db_url = str(session_factory().get_bind().url)
|
||||
|
||||
# Quick check — if lockfile exists and is locked, skip
|
||||
try:
|
||||
check_fd = open(lockfile, "w")
|
||||
fcntl.flock(check_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
fcntl.flock(check_fd, fcntl.LOCK_UN)
|
||||
check_fd.close()
|
||||
except (IOError, OSError):
|
||||
return # another backfill is running for this shop
|
||||
|
||||
pid = os.fork()
|
||||
if pid > 0:
|
||||
os.waitpid(pid, 0)
|
||||
return
|
||||
|
||||
# Intermediate child: detach fully from uWSGI, then fork again
|
||||
os.setsid()
|
||||
pid2 = os.fork()
|
||||
if pid2 > 0:
|
||||
os._exit(0)
|
||||
|
||||
# --- Grandchild process: fully detached from uWSGI ---
|
||||
import resource
|
||||
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
|
||||
if maxfd == resource.RLIM_INFINITY:
|
||||
maxfd = 1024
|
||||
for fd in range(3, maxfd):
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
lock_fd = None
|
||||
try:
|
||||
lock_fd = open(lockfile, "w")
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
lock_fd.write(str(os.getpid()))
|
||||
lock_fd.flush()
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session as SASession
|
||||
from ..models.shop import Shop
|
||||
|
||||
engine = create_engine(db_url)
|
||||
session = SASession(bind=engine)
|
||||
|
||||
def _make_src():
|
||||
return boto3.session.Session().client(
|
||||
"s3",
|
||||
region_name=app_settings["bucket.secure_uploads.region"],
|
||||
endpoint_url=app_settings["bucket.secure_uploads.post_endpoint"],
|
||||
aws_access_key_id=app_settings["bucket.secure_uploads.access_key"],
|
||||
aws_secret_access_key=app_settings["bucket.secure_uploads.secret_key"],
|
||||
)
|
||||
|
||||
try:
|
||||
shop = session.get(Shop, shop_id)
|
||||
if not shop or not shop.has_s3_mirror:
|
||||
return
|
||||
|
||||
src = _make_src()
|
||||
dst = _make_mirror_client(
|
||||
shop.mirror_s3_endpoint,
|
||||
shop.mirror_s3_region,
|
||||
shop.mirror_s3_access_key,
|
||||
shop.mirror_s3_secret_key,
|
||||
)
|
||||
src_bucket = app_settings["bucket.secure_uploads"]
|
||||
dst_bucket = shop.mirror_s3_bucket
|
||||
|
||||
# List all objects under the shop's prefix
|
||||
prefix = f"{shop_id_str}/"
|
||||
continuation_token = None
|
||||
total = 0
|
||||
mirrored = 0
|
||||
|
||||
log.info("Backfill mirror for shop %s (pid %d)", shop.name, os.getpid())
|
||||
|
||||
while True:
|
||||
list_kwargs = {
|
||||
"Bucket": src_bucket,
|
||||
"Prefix": prefix,
|
||||
"MaxKeys": 1000,
|
||||
}
|
||||
if continuation_token:
|
||||
list_kwargs["ContinuationToken"] = continuation_token
|
||||
|
||||
resp = src.list_objects_v2(**list_kwargs)
|
||||
contents = resp.get("Contents", [])
|
||||
total += len(contents)
|
||||
|
||||
for obj in contents:
|
||||
key = obj["Key"]
|
||||
mirror_key(src, src_bucket, key, dst, dst_bucket)
|
||||
mirrored += 1
|
||||
|
||||
if resp.get("IsTruncated"):
|
||||
continuation_token = resp["NextContinuationToken"]
|
||||
else:
|
||||
break
|
||||
|
||||
log.info("Backfill mirror complete for shop %s: %d/%d keys mirrored",
|
||||
shop.name, mirrored, total)
|
||||
|
||||
except Exception:
|
||||
log.exception("Backfill mirror failed for shop %s", shop_id_str)
|
||||
finally:
|
||||
session.close()
|
||||
engine.dispose()
|
||||
except (IOError, OSError):
|
||||
pass # couldn't acquire lock
|
||||
except Exception:
|
||||
log.exception("Backfill mirror child process failed for shop %s", shop_id_str)
|
||||
finally:
|
||||
if lock_fd:
|
||||
try:
|
||||
lock_fd.close()
|
||||
os.unlink(lockfile)
|
||||
except OSError:
|
||||
pass
|
||||
os._exit(0)
|
||||
|
|
@ -104,6 +104,14 @@ class Shop(RBase, Base):
|
|||
unsandbox_public_key = Column(Unicode(128), nullable=True)
|
||||
unsandbox_secret_key = Column(Unicode(128), nullable=True)
|
||||
|
||||
# S3-compatible bucket for mirroring all shop uploads
|
||||
mirror_s3_endpoint = Column(Unicode(256), nullable=True)
|
||||
mirror_s3_region = Column(Unicode(64), nullable=True)
|
||||
mirror_s3_bucket = Column(Unicode(128), nullable=True)
|
||||
mirror_s3_access_key = Column(Unicode(128), nullable=True)
|
||||
mirror_s3_secret_key = Column(Unicode(128), nullable=True)
|
||||
mirror_s3_enabled = Column(Boolean, default=False)
|
||||
|
||||
created_timestamp = Column(BigInteger, nullable=False)
|
||||
updated_timestamp = Column(BigInteger, nullable=False)
|
||||
|
||||
|
|
@ -227,6 +235,17 @@ class Shop(RBase, Base):
|
|||
us.role_id = role_id
|
||||
return us
|
||||
|
||||
@property
|
||||
def has_s3_mirror(self):
|
||||
"""Return True if shop has S3 mirror bucket configured and enabled."""
|
||||
return bool(
|
||||
self.mirror_s3_enabled
|
||||
and self.mirror_s3_endpoint
|
||||
and self.mirror_s3_bucket
|
||||
and self.mirror_s3_access_key
|
||||
and self.mirror_s3_secret_key
|
||||
)
|
||||
|
||||
@property
|
||||
def owners(self):
|
||||
"""Returns a list of user objects who have the owner role on this shop."""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
"""add S3 mirror bucket credentials to shop
|
||||
|
||||
Revision ID: 6b516114c393
|
||||
Revises: f898ba460612
|
||||
Create Date: 2026-02-27 15:43:58.571158
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '6b516114c393'
|
||||
down_revision = 'f898ba460612'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
def _column_exists(table, column):
|
||||
conn = op.get_bind()
|
||||
result = conn.execute(sa.text(f"PRAGMA table_info({table})"))
|
||||
return any(row[1] == column for row in result.fetchall())
|
||||
|
||||
|
||||
def upgrade():
|
||||
columns = [
|
||||
("mirror_s3_endpoint", sa.Unicode(256), True, None),
|
||||
("mirror_s3_region", sa.Unicode(64), True, None),
|
||||
("mirror_s3_bucket", sa.Unicode(128), True, None),
|
||||
("mirror_s3_access_key", sa.Unicode(128), True, None),
|
||||
("mirror_s3_secret_key", sa.Unicode(128), True, None),
|
||||
("mirror_s3_enabled", sa.Boolean(), False, "0"),
|
||||
]
|
||||
for col_name, col_type, nullable, server_default in columns:
|
||||
if not _column_exists("mps_shop", col_name):
|
||||
op.add_column(
|
||||
"mps_shop",
|
||||
sa.Column(
|
||||
col_name,
|
||||
col_type,
|
||||
nullable=nullable,
|
||||
server_default=server_default,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
for col_name in (
|
||||
"mirror_s3_enabled",
|
||||
"mirror_s3_secret_key",
|
||||
"mirror_s3_access_key",
|
||||
"mirror_s3_bucket",
|
||||
"mirror_s3_region",
|
||||
"mirror_s3_endpoint",
|
||||
):
|
||||
if _column_exists("mps_shop", col_name):
|
||||
op.drop_column("mps_shop", col_name)
|
||||
|
|
@ -470,6 +470,123 @@
|
|||
<br />
|
||||
<br />
|
||||
|
||||
<section class="one-column">
|
||||
<section class="shop-settings well">
|
||||
|
||||
<h3>S3 Mirror Bucket</h3>
|
||||
<p>Connect an S3-compatible bucket to automatically mirror all uploaded files. Works with DigitalOcean Spaces, AWS S3, MinIO, Backblaze B2, and any S3-compatible service. Your MPS bucket remains the origin — this is a backup/mirror.</p>
|
||||
|
||||
<form method="post" action="/s/{{ request.shop.id }}/settings" onsubmit="submit.disabled = true; return true;">
|
||||
<input type="hidden" name="form_section" value="mirror-settings" />
|
||||
|
||||
<details class="api-key-details">
|
||||
<summary class="toggle-summary">Show Mirror Bucket Settings</summary>
|
||||
|
||||
<div class="api-key-fields">
|
||||
|
||||
<br />
|
||||
|
||||
<label for="mirror_s3_enabled_checkbox">
|
||||
<input type="checkbox" name="mirror_s3_enabled_checkbox" id="mirror_s3_enabled_checkbox"
|
||||
{% if mirror_s3_enabled %}checked{% endif %} />
|
||||
Enable mirror sync
|
||||
</label>
|
||||
|
||||
<br />
|
||||
<br />
|
||||
|
||||
<label for="mirror_s3_endpoint_input">S3 Endpoint URL</label>
|
||||
<input
|
||||
name = "mirror_s3_endpoint"
|
||||
type = "url"
|
||||
id = "mirror_s3_endpoint_input"
|
||||
class = "full-width-input"
|
||||
value = "{{ mirror_s3_endpoint }}"
|
||||
placeholder = "https://nyc3.digitaloceanspaces.com" />
|
||||
|
||||
<br />
|
||||
<br />
|
||||
|
||||
<label for="mirror_s3_region_input">Region (optional)</label>
|
||||
<input
|
||||
name = "mirror_s3_region"
|
||||
type = "text"
|
||||
id = "mirror_s3_region_input"
|
||||
class = "full-width-input"
|
||||
value = "{{ mirror_s3_region }}"
|
||||
placeholder = "nyc3" />
|
||||
|
||||
<br />
|
||||
<br />
|
||||
|
||||
<label for="mirror_s3_bucket_input">Bucket Name</label>
|
||||
<input
|
||||
name = "mirror_s3_bucket"
|
||||
type = "text"
|
||||
id = "mirror_s3_bucket_input"
|
||||
class = "full-width-input"
|
||||
value = "{{ mirror_s3_bucket }}"
|
||||
placeholder = "my-shop-backup" />
|
||||
|
||||
<br />
|
||||
<br />
|
||||
|
||||
<label for="mirror_s3_access_key_input">Access Key</label>
|
||||
<input
|
||||
name = "mirror_s3_access_key"
|
||||
type = "text"
|
||||
id = "mirror_s3_access_key_input"
|
||||
class = "full-width-input"
|
||||
value = "{{ mirror_s3_access_key }}"
|
||||
placeholder = "Access Key ID" />
|
||||
|
||||
<br />
|
||||
<br />
|
||||
|
||||
<label for="mirror_s3_secret_key_input">Secret Key</label>
|
||||
<input
|
||||
name = "mirror_s3_secret_key"
|
||||
type = "password"
|
||||
id = "mirror_s3_secret_key_input"
|
||||
class = "full-width-input"
|
||||
value = "{{ mirror_s3_secret_key }}"
|
||||
placeholder = "Secret Access Key" />
|
||||
|
||||
<br />
|
||||
<br />
|
||||
|
||||
{% if request.shop.has_s3_mirror %}
|
||||
<small class="success-indicator">Mirror configured and enabled</small>
|
||||
{% elif request.shop.mirror_s3_bucket %}
|
||||
<small class="status-message">Mirror credentials saved but not enabled. Check the box above to activate.</small>
|
||||
{% else %}
|
||||
<small class="status-message">Add S3-compatible bucket credentials to enable file mirroring.</small>
|
||||
{% endif %}
|
||||
|
||||
<br />
|
||||
<br />
|
||||
|
||||
<input type="submit" name="submit" class="mps-submit" value="Save Mirror Settings" />
|
||||
|
||||
</div>
|
||||
</details>
|
||||
</form>
|
||||
|
||||
{% if request.shop.has_s3_mirror %}
|
||||
<form method="post" action="/s/{{ request.shop.id }}/settings" onsubmit="this.querySelector('[type=submit]').disabled = true; return true;">
|
||||
<input type="hidden" name="form_section" value="backfill-s3-mirror" />
|
||||
<input type="submit" class="mps-submit" value="Backfill Mirror Bucket" />
|
||||
<br />
|
||||
<small>Copy all existing files to your mirror bucket. Runs in background.</small>
|
||||
</form>
|
||||
{% endif %}
|
||||
|
||||
</section>
|
||||
</section>
|
||||
|
||||
<br />
|
||||
<br />
|
||||
|
||||
{% if request.monero_enabled %}
|
||||
<section class="one-column">
|
||||
<section class="shop-settings well">
|
||||
|
|
|
|||
|
|
@ -4044,3 +4044,113 @@ class TestAnalytics(AuthenticatedFunctionalTests):
|
|||
# Should NOT contain media preview elements
|
||||
self.assertNotIn("edit-media-preview", body)
|
||||
self.assertNotIn("<video", body)
|
||||
|
||||
def test_shop_mirror_default_disabled(self):
|
||||
"""Test that new shops have no mirror bucket configured."""
|
||||
shop = self._create_shop_helper(
|
||||
user_creds=self.user1_creds, shop_params=self.shop1_params
|
||||
)
|
||||
self.dbsession.refresh(shop)
|
||||
self.assertIsNone(shop.mirror_s3_endpoint)
|
||||
self.assertIsNone(shop.mirror_s3_bucket)
|
||||
self.assertFalse(shop.mirror_s3_enabled)
|
||||
self.assertFalse(shop.has_s3_mirror)
|
||||
|
||||
def test_shop_mirror_settings_save(self):
|
||||
"""Test saving S3 mirror bucket credentials via settings form."""
|
||||
shop = self._create_shop_helper(
|
||||
user_creds=self.user1_creds, shop_params=self.shop1_params
|
||||
)
|
||||
|
||||
res = self.testapp.post(
|
||||
f"/s/{shop.id}/settings",
|
||||
{
|
||||
"form_section": "mirror-settings",
|
||||
"mirror_s3_endpoint": "https://nyc3.digitaloceanspaces.com",
|
||||
"mirror_s3_region": "nyc3",
|
||||
"mirror_s3_bucket": "my-shop-backup",
|
||||
"mirror_s3_access_key": "AKIAIOSFODNN7EXAMPLE",
|
||||
"mirror_s3_secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"mirror_s3_enabled_checkbox": "off",
|
||||
"submit": "Save Mirror Settings",
|
||||
},
|
||||
status=302,
|
||||
)
|
||||
res = res.follow()
|
||||
self.assertIn("Mirror credentials saved", res.text)
|
||||
|
||||
self.dbsession.expire(shop)
|
||||
self.assertEqual(shop.mirror_s3_endpoint, "https://nyc3.digitaloceanspaces.com")
|
||||
self.assertEqual(shop.mirror_s3_region, "nyc3")
|
||||
self.assertEqual(shop.mirror_s3_bucket, "my-shop-backup")
|
||||
self.assertEqual(shop.mirror_s3_access_key, "AKIAIOSFODNN7EXAMPLE")
|
||||
self.assertEqual(shop.mirror_s3_secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
|
||||
self.assertFalse(shop.mirror_s3_enabled)
|
||||
# Not enabled, so has_s3_mirror is False
|
||||
self.assertFalse(shop.has_s3_mirror)
|
||||
|
||||
def test_shop_mirror_settings_clear(self):
|
||||
"""Test clearing S3 mirror bucket credentials."""
|
||||
shop = self._create_shop_helper(
|
||||
user_creds=self.user1_creds, shop_params=self.shop1_params
|
||||
)
|
||||
|
||||
# First save some credentials
|
||||
self.testapp.post(
|
||||
f"/s/{shop.id}/settings",
|
||||
{
|
||||
"form_section": "mirror-settings",
|
||||
"mirror_s3_endpoint": "https://nyc3.digitaloceanspaces.com",
|
||||
"mirror_s3_region": "nyc3",
|
||||
"mirror_s3_bucket": "my-shop-backup",
|
||||
"mirror_s3_access_key": "AKIAIOSFODNN7EXAMPLE",
|
||||
"mirror_s3_secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"submit": "Save Mirror Settings",
|
||||
},
|
||||
status=302,
|
||||
)
|
||||
|
||||
# Clear by submitting empty fields
|
||||
res = self.testapp.post(
|
||||
f"/s/{shop.id}/settings",
|
||||
{
|
||||
"form_section": "mirror-settings",
|
||||
"mirror_s3_endpoint": "",
|
||||
"mirror_s3_region": "",
|
||||
"mirror_s3_bucket": "",
|
||||
"mirror_s3_access_key": "",
|
||||
"mirror_s3_secret_key": "",
|
||||
"submit": "Save Mirror Settings",
|
||||
},
|
||||
status=302,
|
||||
)
|
||||
res = res.follow()
|
||||
self.assertIn("Mirror storage credentials cleared", res.text)
|
||||
|
||||
self.dbsession.expire(shop)
|
||||
self.assertIsNone(shop.mirror_s3_endpoint)
|
||||
self.assertIsNone(shop.mirror_s3_bucket)
|
||||
self.assertFalse(shop.mirror_s3_enabled)
|
||||
|
||||
def test_shop_mirror_settings_requires_fields(self):
|
||||
"""Test that mirror settings validation requires all credential fields."""
|
||||
shop = self._create_shop_helper(
|
||||
user_creds=self.user1_creds, shop_params=self.shop1_params
|
||||
)
|
||||
|
||||
# Submit with endpoint but missing other required fields
|
||||
res = self.testapp.post(
|
||||
f"/s/{shop.id}/settings",
|
||||
{
|
||||
"form_section": "mirror-settings",
|
||||
"mirror_s3_endpoint": "https://nyc3.digitaloceanspaces.com",
|
||||
"mirror_s3_region": "",
|
||||
"mirror_s3_bucket": "",
|
||||
"mirror_s3_access_key": "",
|
||||
"mirror_s3_secret_key": "",
|
||||
"submit": "Save Mirror Settings",
|
||||
},
|
||||
status=302,
|
||||
)
|
||||
res = res.follow()
|
||||
self.assertIn("Endpoint, bucket, access key, and secret key are all required", res.text)
|
||||
|
|
|
|||
|
|
@ -339,6 +339,17 @@ def product_edit(request):
|
|||
MetadataDirective="REPLACE",
|
||||
)
|
||||
|
||||
# Mirror to shop's custom S3 bucket if configured
|
||||
from ..lib.s3_mirror import mirror_key_async
|
||||
mirror_key_async(
|
||||
request.secure_uploads_client,
|
||||
request.app["bucket.secure_uploads"],
|
||||
f"{product.s3_path}/{file_key}",
|
||||
product.shop,
|
||||
content_type=product.get_content_type(file_key),
|
||||
cache_control=cache_control,
|
||||
)
|
||||
|
||||
# delete original upload key.
|
||||
request.secure_uploads_client.delete_object(
|
||||
Bucket=s3_webhook_bucket,
|
||||
|
|
@ -392,6 +403,16 @@ def product_edit(request):
|
|||
request.dbsession.flush()
|
||||
product.update_s3_acls(request.secure_uploads_client, request.app["bucket.secure_uploads"])
|
||||
|
||||
# Mirror karaoke tracks to shop's custom bucket
|
||||
if shop.has_s3_mirror:
|
||||
from ..lib.s3_mirror import mirror_keys_async
|
||||
mirror_keys_async(
|
||||
request.secure_uploads_client,
|
||||
request.app["bucket.secure_uploads"],
|
||||
[f"{product.s3_path}/instrumentals", f"{product.s3_path}/vocals"],
|
||||
shop,
|
||||
)
|
||||
|
||||
# redirect back to this page to clear
|
||||
# the params posted by the s3 webhooks.
|
||||
return HTTPFound(f"{product.absolute_edit_url(request)}?uploaded={file_key}")
|
||||
|
|
|
|||
|
|
@ -836,6 +836,62 @@ def shop_settings(request):
|
|||
else:
|
||||
request.session.flash(("Set unsandbox API keys first.", "error"))
|
||||
|
||||
# Handle S3 mirror bucket settings
|
||||
if form_section == "mirror-settings":
|
||||
mirror_s3_endpoint = request.params.get("mirror_s3_endpoint", "").strip()
|
||||
mirror_s3_region = request.params.get("mirror_s3_region", "").strip()
|
||||
mirror_s3_bucket = request.params.get("mirror_s3_bucket", "").strip()
|
||||
mirror_s3_access_key = request.params.get("mirror_s3_access_key", "").strip()
|
||||
mirror_s3_secret_key = request.params.get("mirror_s3_secret_key", "").strip()
|
||||
mirror_s3_enabled = checkbox_to_bool(
|
||||
request.params.get("mirror_s3_enabled_checkbox", "off")
|
||||
)
|
||||
|
||||
# If all credential fields are empty, clear everything
|
||||
if not any([mirror_s3_endpoint, mirror_s3_bucket, mirror_s3_access_key, mirror_s3_secret_key]):
|
||||
shop.mirror_s3_endpoint = None
|
||||
shop.mirror_s3_region = None
|
||||
shop.mirror_s3_bucket = None
|
||||
shop.mirror_s3_access_key = None
|
||||
shop.mirror_s3_secret_key = None
|
||||
shop.mirror_s3_enabled = False
|
||||
request.session.flash(("Mirror storage credentials cleared.", "success"))
|
||||
elif mirror_s3_endpoint and not mirror_s3_endpoint.startswith("http"):
|
||||
request.session.flash(("Mirror S3 endpoint must start with http:// or https://", "error"))
|
||||
elif not (mirror_s3_endpoint and mirror_s3_bucket and mirror_s3_access_key and mirror_s3_secret_key):
|
||||
request.session.flash(("Endpoint, bucket, access key, and secret key are all required.", "error"))
|
||||
else:
|
||||
shop.mirror_s3_endpoint = mirror_s3_endpoint
|
||||
shop.mirror_s3_region = mirror_s3_region or None
|
||||
shop.mirror_s3_bucket = mirror_s3_bucket
|
||||
shop.mirror_s3_access_key = mirror_s3_access_key
|
||||
shop.mirror_s3_secret_key = mirror_s3_secret_key
|
||||
shop.mirror_s3_enabled = mirror_s3_enabled
|
||||
|
||||
# Test connection if enabled
|
||||
if mirror_s3_enabled:
|
||||
from ..lib.s3_mirror import test_mirror_connection
|
||||
ok, err = test_mirror_connection(shop)
|
||||
if ok:
|
||||
request.session.flash(("Mirror bucket configured and connection verified.", "success"))
|
||||
else:
|
||||
request.session.flash((f"Mirror credentials saved but connection test failed: {err}", "error"))
|
||||
else:
|
||||
request.session.flash(("Mirror credentials saved. Enable the checkbox to activate sync.", "success"))
|
||||
|
||||
# Handle backfill S3 mirror button
|
||||
if form_section == "backfill-s3-mirror":
|
||||
if shop.has_s3_mirror:
|
||||
from ..lib.s3_mirror import backfill_mirror_async
|
||||
backfill_mirror_async(
|
||||
shop.id,
|
||||
request.registry["dbsession_factory"],
|
||||
request.app,
|
||||
)
|
||||
request.session.flash(("Mirroring existing files to your bucket in background.", "success"))
|
||||
else:
|
||||
request.session.flash(("Configure and enable mirror bucket credentials first.", "error"))
|
||||
|
||||
# Handle maintenance settings form
|
||||
if form_section == "maintenance-settings":
|
||||
if shop.maint_mode != maint_mode:
|
||||
|
|
@ -1016,6 +1072,17 @@ def shop_settings(request):
|
|||
MetadataDirective="REPLACE",
|
||||
)
|
||||
|
||||
# Mirror shop asset to custom S3 bucket if configured
|
||||
from ..lib.s3_mirror import mirror_key_async
|
||||
mirror_key_async(
|
||||
request.secure_uploads_client,
|
||||
request.app["bucket.secure_uploads"],
|
||||
f"{shop.id}/meta/{file_key}",
|
||||
shop,
|
||||
content_type=content_type,
|
||||
cache_control=cache_control,
|
||||
)
|
||||
|
||||
# delete original upload key.
|
||||
request.secure_uploads_client.delete_object(
|
||||
Bucket=s3_webhook_bucket,
|
||||
|
|
@ -1134,6 +1201,12 @@ def shop_settings(request):
|
|||
"doge_processor": doge_processor,
|
||||
"unsandbox_public_key": shop.unsandbox_public_key or "",
|
||||
"unsandbox_secret_key": shop.unsandbox_secret_key or "",
|
||||
"mirror_s3_endpoint": shop.mirror_s3_endpoint or "",
|
||||
"mirror_s3_region": shop.mirror_s3_region or "",
|
||||
"mirror_s3_bucket": shop.mirror_s3_bucket or "",
|
||||
"mirror_s3_access_key": shop.mirror_s3_access_key or "",
|
||||
"mirror_s3_secret_key": shop.mirror_s3_secret_key or "",
|
||||
"mirror_s3_enabled": shop.mirror_s3_enabled,
|
||||
"signed_posts": signed_posts,
|
||||
"get_endpoints": get_endpoints,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue