From 4d899d367f21fe4b9e289c76a4e2113109172569 Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Mon, 9 Jun 2025 01:09:25 +0000 Subject: [PATCH] Introduced support for scheduling actions --- app.py | 758 +++++++++++++++++++++- migration_scripts/01_visibility_column.py | 243 +++++++ templates/add_scheduled_action.html.j2 | 28 + templates/base.html.j2 | 7 + templates/edit_media.html.j2 | 31 +- templates/edit_scheduled_action.html.j2 | 30 + templates/home.html.j2 | 124 +++- templates/upload_media.html.j2 | 38 +- templates/view_media_details.html.j2 | 38 +- 9 files changed, 1238 insertions(+), 59 deletions(-) create mode 100644 migration_scripts/01_visibility_column.py create mode 100644 templates/add_scheduled_action.html.j2 create mode 100644 templates/edit_scheduled_action.html.j2 diff --git a/app.py b/app.py index faaa4ce..da5e29a 100644 --- a/app.py +++ b/app.py @@ -1,5 +1,5 @@ ############################################################################### -# app.py - PyraFiles Application with Proper Agent and User Separation +# app.py - Enhanced PyraFiles Application with Scheduled Actions and More ############################################################################### import os import base64 @@ -15,6 +15,10 @@ import mimetypes import json import logging import unicodedata +import threading +import time +import signal +import sys from email.mime.text import MIMEText from pyramid.config import Configurator @@ -33,6 +37,7 @@ from sqlalchemy import ( ForeignKey, Text, or_, + inspect, ) from sqlalchemy.orm import ( declarative_base, @@ -181,6 +186,19 @@ def get_mime_type(filename): return mime_type +def is_ascii_file(mime_type): + """Check if the file is ASCII-based (text files).""" + ascii_types = [ + "text/", + "application/json", + "application/xml", + "application/javascript", + "application/x-yaml", + "application/yaml", + ] + return any(mime_type.startswith(t) for t in ascii_types) + + def uuid_to_short_id(u): """Encode UUID to a URL-safe base64 string without padding.""" return base64.urlsafe_b64encode(u.bytes).decode("ascii").rstrip("=") @@ -351,6 +369,15 @@ class Agent(Base): ) +# New table for scheduler lock +class SchedulerLock(Base): + __tablename__ = "scheduler_lock" + id = Column(Integer, primary_key=True) + locked = Column(Boolean, default=False) + locked_at = Column(DateTime, nullable=True) + process_id = Column(String, nullable=True) + + ################################################################################ # Namespace Database Models ################################################################################ @@ -365,15 +392,33 @@ class Media(NamespaceBase): filename = Column(String, nullable=False) title = Column(String, nullable=True) # Optional title media_type = Column(String, nullable=False) # 'image', 'audio', 'video' + mime_type = Column(String, nullable=True) # MIME type for compatibility media_b64 = Column(Text, nullable=False) upload_date = Column(DateTime, default=datetime.datetime.utcnow) - is_public = Column(Boolean, default=True) + visibility = Column(String, default="public") # 'public', 'private', 'unlisted' size = Column(Integer, nullable=False) # Size in bytes def __repr__(self): return f"" +class ScheduledAction(NamespaceBase): + __tablename__ = "scheduled_actions" + id = Column(String, primary_key=True) # UUID + media_id = Column(String, ForeignKey("media.id"), nullable=False) + action_type = Column( + String, nullable=False + ) # 'set_public', 'set_private', 'set_unlisted', 'delete' + scheduled_date = Column(DateTime, nullable=False) + created_at = Column(DateTime, default=datetime.datetime.utcnow) + status = Column(String, default="pending") # 'pending', 'completed', 'failed' + + media = relationship("Media") + + def __repr__(self): + return f"" + + ################################################################################ # Jinja2 Environment and Custom Filters ################################################################################ @@ -572,6 +617,208 @@ def reader_required(view_func): return wrapper +################################################################################ +# Scheduler Functions +################################################################################ + + +def acquire_scheduler_lock(dbsession): + """Acquire the scheduler lock.""" + process_id = f"{os.getpid()}_{int(time.time())}" + + # Try to get existing lock + lock = dbsession.query(SchedulerLock).first() + + if not lock: + # Create new lock + lock = SchedulerLock( + locked=True, locked_at=datetime.datetime.utcnow(), process_id=process_id + ) + dbsession.add(lock) + dbsession.commit() + return True + + # Check if lock is stale (older than 5 minutes) + if lock.locked and lock.locked_at: + if datetime.datetime.utcnow() - lock.locked_at > datetime.timedelta(minutes=5): + log.warning("Detected stale scheduler lock, releasing it") + lock.locked = False + lock.locked_at = None + lock.process_id = None + dbsession.commit() + + if not lock.locked: + lock.locked = True + lock.locked_at = datetime.datetime.utcnow() + lock.process_id = process_id + dbsession.commit() + return True + + return False + + +def release_scheduler_lock(dbsession): + """Release the scheduler lock.""" + lock = dbsession.query(SchedulerLock).first() + if lock and lock.locked: + lock.locked = False + lock.locked_at = None + lock.process_id = None + dbsession.commit() + + +def process_scheduled_actions(): + """Process all scheduled actions across all namespaces.""" + main_engine = create_engine( + DB_URL, + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + MainSessionFactory = sessionmaker(bind=main_engine) + main_dbsession = MainSessionFactory() + + try: + if not acquire_scheduler_lock(main_dbsession): + return # Another process is already running + + log.info("Scheduler acquired lock, processing scheduled actions...") + + # Get all namespaces + namespaces = main_dbsession.query(Namespace).all() + + for namespace in namespaces: + process_namespace_scheduled_actions(namespace.id) + + log.info("Finished processing scheduled actions") + + except Exception as e: + log.error(f"Error in scheduler: {e}") + finally: + release_scheduler_lock(main_dbsession) + main_dbsession.close() + main_engine.dispose() + + +def process_namespace_scheduled_actions(namespace_id): + """Process scheduled actions for a specific namespace.""" + namespace_db_url = get_namespace_db_url(namespace_id) + + try: + namespace_engine = create_engine( + namespace_db_url, + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + NamespaceSessionFactory = sessionmaker(bind=namespace_engine) + namespace_dbsession = NamespaceSessionFactory() + + # Check if scheduled_actions table exists + inspector = inspect(namespace_engine) + if "scheduled_actions" not in inspector.get_table_names(): + log.debug( + f"No scheduled_actions table in namespace {namespace_id}, skipping" + ) + return + + # Get pending actions that are due + now = datetime.datetime.utcnow() + pending_actions = ( + namespace_dbsession.query(ScheduledAction) + .filter( + ScheduledAction.status == "pending", + ScheduledAction.scheduled_date <= now, + ) + .all() + ) + + for action in pending_actions: + try: + media = ( + namespace_dbsession.query(Media) + .filter_by(id=action.media_id) + .first() + ) + if not media: + action.status = "failed" + continue + + if action.action_type == "delete": + namespace_dbsession.delete(media) + log.info(f"Scheduled delete executed for media {media.filename}") + elif action.action_type == "set_public": + media.visibility = "public" + log.info( + f"Scheduled set_public executed for media {media.filename}" + ) + elif action.action_type == "set_private": + media.visibility = "private" + log.info( + f"Scheduled set_private executed for media {media.filename}" + ) + elif action.action_type == "set_unlisted": + media.visibility = "unlisted" + log.info( + f"Scheduled set_unlisted executed for media {media.filename}" + ) + + action.status = "completed" + + except Exception as e: + log.error(f"Error processing scheduled action {action.id}: {e}") + action.status = "failed" + + namespace_dbsession.commit() + + except Exception as e: + log.error(f"Error processing namespace {namespace_id}: {e}") + finally: + if "namespace_dbsession" in locals(): + namespace_dbsession.close() + if "namespace_engine" in locals(): + namespace_engine.dispose() + + +# Global variables for scheduler thread +scheduler_thread = None +scheduler_running = False + + +def scheduler_worker(): + """Background scheduler worker that runs every minute.""" + global scheduler_running + while scheduler_running: + try: + process_scheduled_actions() + except Exception as e: + log.error(f"Scheduler worker error: {e}") + + # Sleep for 60 seconds + for i in range(60): + if not scheduler_running: + break + time.sleep(1) + + +def start_scheduler(): + """Start the background scheduler.""" + global scheduler_thread, scheduler_running + if not scheduler_running: + scheduler_running = True + scheduler_thread = threading.Thread(target=scheduler_worker, daemon=True) + scheduler_thread.start() + log.info("Scheduler started") + + +def stop_scheduler(): + """Stop the background scheduler.""" + global scheduler_thread, scheduler_running + if scheduler_running: + scheduler_running = False + if scheduler_thread: + scheduler_thread.join(timeout=5) + log.info("Scheduler stopped") + + ################################################################################ # Routes and Views ################################################################################ @@ -1202,19 +1449,93 @@ def upload_media_post_view(request): media_id = str(media_uuid) media_short_id = uuid_to_short_id(media_uuid) - is_public = request.POST.get("is_public") == "on" + visibility = request.POST.get("visibility", "public") + if visibility not in ["public", "private", "unlisted"]: + visibility = "public" media = Media( id=media_id, short_id=media_short_id, filename=filename, - title=title, # now has a fallback of filename + title=title, media_type=media_type, + mime_type=mime_type, media_b64=encoded_str, - is_public=is_public, + visibility=visibility, size=file_size, ) namespace_dbsession.add(media) + + # Handle scheduled actions + scheduled_public = request.POST.get("scheduled_public") + scheduled_private = request.POST.get("scheduled_private") + scheduled_unlisted = request.POST.get("scheduled_unlisted") + scheduled_delete = request.POST.get("scheduled_delete") + + # Check if scheduled_actions table exists before adding scheduled actions + try: + inspector = inspect(namespace_dbsession.bind) + has_scheduled_actions = "scheduled_actions" in inspector.get_table_names() + except: + has_scheduled_actions = False + + if has_scheduled_actions: + if scheduled_public: + try: + scheduled_date = datetime.datetime.fromisoformat(scheduled_public) + action_uuid = uuid.uuid4() + action = ScheduledAction( + id=str(action_uuid), + media_id=media_id, + action_type="set_public", + scheduled_date=scheduled_date, + ) + namespace_dbsession.add(action) + except ValueError: + pass + + if scheduled_private: + try: + scheduled_date = datetime.datetime.fromisoformat(scheduled_private) + action_uuid = uuid.uuid4() + action = ScheduledAction( + id=str(action_uuid), + media_id=media_id, + action_type="set_private", + scheduled_date=scheduled_date, + ) + namespace_dbsession.add(action) + except ValueError: + pass + + if scheduled_unlisted: + try: + scheduled_date = datetime.datetime.fromisoformat(scheduled_unlisted) + action_uuid = uuid.uuid4() + action = ScheduledAction( + id=str(action_uuid), + media_id=media_id, + action_type="set_unlisted", + scheduled_date=scheduled_date, + ) + namespace_dbsession.add(action) + except ValueError: + pass + + if scheduled_delete: + try: + scheduled_date = datetime.datetime.fromisoformat(scheduled_delete) + action_uuid = uuid.uuid4() + action = ScheduledAction( + id=str(action_uuid), + media_id=media_id, + action_type="delete", + scheduled_date=scheduled_date, + ) + namespace_dbsession.add(action) + except ValueError: + pass + namespace_dbsession.flush() return HTTPFound( @@ -1236,24 +1557,37 @@ def list_media_view(request): return HTTPForbidden("You do not have access to this namespace.") # Get the media items, excluding the media_b64 column - media_items = ( - namespace_dbsession.query(Media) - .options( - load_only( - Media.id, - Media.short_id, - Media.filename, - Media.title, - Media.media_type, - Media.upload_date, - Media.is_public, - Media.size, - ) + # Only show public media unless user has editor+ access + query = namespace_dbsession.query(Media).options( + load_only( + Media.id, + Media.short_id, + Media.filename, + Media.title, + Media.media_type, + Media.upload_date, + Media.visibility, + Media.size, ) - .order_by(Media.upload_date.desc()) - .all() ) + # Filter based on user permissions + user_role = get_user_or_agent_namespace_role(request) + if user_role in ["owner", "editor"]: + # Owners and editors can see all media except unlisted + media_items = ( + query.filter(Media.visibility != "unlisted") + .order_by(Media.upload_date.desc()) + .all() + ) + else: + # Readers and public users can only see public media + media_items = ( + query.filter(Media.visibility == "public") + .order_by(Media.upload_date.desc()) + .all() + ) + return { "request": request, "namespace": namespace, @@ -1274,15 +1608,58 @@ def view_media_details_view(request): role = get_user_or_agent_namespace_role(request) is_owner_or_editor = role in ["owner", "editor"] - # Check if media is public or user has access - if not media.is_public and not check_namespace_permission(request, "reader"): + # Check visibility permissions + if media.visibility == "private" and not check_namespace_permission( + request, "reader" + ): return Response("Media not available.", status=403) + elif media.visibility == "unlisted" and not is_owner_or_editor: + return Response("Media not available.", status=403) + elif media.visibility == "public": + # Public media is always accessible + pass + + # Check if this is an ASCII file and get preview content + ascii_preview = None + mime_type = get_mime_type(media.filename) + if is_ascii_file(mime_type): + try: + raw_content = base64.b64decode(media.media_b64).decode("utf-8") + lines = raw_content.split("\n") + if len(lines) > 100: + ascii_preview = "\n".join(lines[:100]) + "\n... (truncated)" + else: + ascii_preview = raw_content + except (UnicodeDecodeError, Exception): + ascii_preview = None + + # Get scheduled actions for this media + scheduled_actions = [] + if is_owner_or_editor: + try: + # Check if scheduled_actions table exists + inspector = inspect(namespace_dbsession.bind) + if "scheduled_actions" in inspector.get_table_names(): + scheduled_actions = ( + namespace_dbsession.query(ScheduledAction) + .filter( + ScheduledAction.media_id == media.id, + ScheduledAction.status == "pending", + ) + .order_by(ScheduledAction.scheduled_date) + .all() + ) + except Exception as e: + log.debug(f"Could not query scheduled actions: {e}") + scheduled_actions = [] return { "request": request, "media": media, "namespace": namespace, "is_owner_or_editor": is_owner_or_editor, + "ascii_preview": ascii_preview, + "scheduled_actions": scheduled_actions, } @@ -1297,6 +1674,20 @@ def delete_media_view(request): if not media: return Response("Media not found.", status=404) + # Delete associated scheduled actions if table exists + try: + inspector = inspect(namespace_dbsession.bind) + if "scheduled_actions" in inspector.get_table_names(): + scheduled_actions = ( + namespace_dbsession.query(ScheduledAction) + .filter_by(media_id=media.id) + .all() + ) + for action in scheduled_actions: + namespace_dbsession.delete(action) + except Exception as e: + log.debug(f"Could not delete scheduled actions: {e}") + namespace_dbsession.delete(media) namespace_dbsession.flush() @@ -1318,10 +1709,20 @@ def edit_media_get_view(request): if not media: return Response("Media not found.", status=404) + # Check if this is an ASCII file and get editable content + ascii_content = None + mime_type = get_mime_type(media.filename) + if is_ascii_file(mime_type): + try: + ascii_content = base64.b64decode(media.media_b64).decode("utf-8") + except (UnicodeDecodeError, Exception): + ascii_content = None + return { "request": request, "media": media, "namespace": namespace, + "ascii_content": ascii_content, } @@ -1340,6 +1741,20 @@ def edit_media_post_view(request): new_title = request.POST.get("title", "").strip() media.title = new_title + # Handle ASCII content update + ascii_content = request.POST.get("ascii_content", "").strip() + mime_type = get_mime_type(media.filename) + if ascii_content and is_ascii_file(mime_type): + try: + # Update the media content with the new ASCII content + encoded_str = base64.b64encode(ascii_content.encode("utf-8")).decode( + "utf-8" + ) + media.media_b64 = encoded_str + media.size = len(ascii_content.encode("utf-8")) + except Exception as e: + return Response(f"Error updating content: {e}", status=400) + # Handle media file update new_media_file = request.POST.get("media_file") if new_media_file is not None and getattr(new_media_file, "filename", "").strip(): @@ -1358,12 +1773,14 @@ def edit_media_post_view(request): media.filename = filename media.media_type = media_type + media.mime_type = mime_type # Update mime_type too media.media_b64 = encoded_str media.size = file_size - # Handle public/private update - is_public = request.POST.get("is_public") == "on" - media.is_public = is_public + # Handle visibility update + visibility = request.POST.get("visibility", "public") + if visibility in ["public", "private", "unlisted"]: + media.visibility = visibility namespace_dbsession.flush() @@ -1391,9 +1808,21 @@ def view_media_view(request): if not media: return Response("Media not found.", status=404) - # Check if media is public or user has access - if not media.is_public and not check_namespace_permission(request, "reader"): + role = get_user_or_agent_namespace_role(request) + is_owner_or_editor = role in ["owner", "editor"] + + # Check visibility permissions + if media.visibility == "private" and not check_namespace_permission( + request, "reader" + ): return Response("Media not available.", status=403) + elif media.visibility == "unlisted": + # Unlisted media is accessible via direct link but not listed + if not is_owner_or_editor and not check_namespace_permission(request, "reader"): + return Response("Media not available.", status=403) + elif media.visibility == "public": + # Public media is always accessible via direct link + pass media_data = base64.b64decode(media.media_b64) mime_type = get_mime_type(media.filename) @@ -1423,6 +1852,248 @@ def view_media_view(request): return response +################################################################################ +# Scheduled Action Management +################################################################################ + + +@view_config(route_name="delete_scheduled_action", request_method="POST") +@editor_required +def delete_scheduled_action_view(request): + namespace = request.namespace + namespace_dbsession = request.namespace_dbsession + media_short_id = request.matchdict.get("media_short_id") + action_id = request.matchdict.get("action_id") + + # Verify media exists + media = namespace_dbsession.query(Media).filter_by(short_id=media_short_id).first() + if not media: + return Response("Media not found.", status=404) + + # Check if scheduled_actions table exists + try: + inspector = inspect(namespace_dbsession.bind) + if "scheduled_actions" not in inspector.get_table_names(): + return Response("Scheduled actions not supported.", status=404) + except: + return Response("Scheduled actions not supported.", status=404) + + # Find and delete the scheduled action + scheduled_action = ( + namespace_dbsession.query(ScheduledAction) + .filter(ScheduledAction.id == action_id, ScheduledAction.media_id == media.id) + .first() + ) + + if not scheduled_action: + return Response("Scheduled action not found.", status=404) + + namespace_dbsession.delete(scheduled_action) + namespace_dbsession.flush() + + request.session.flash( + f"Scheduled {scheduled_action.action_type.replace('set_', '').replace('_', ' ')} action deleted." + ) + + return HTTPFound( + location=request.route_url( + "view_media_details", + namespace_short_id=namespace.short_id, + media_short_id=media.short_id, + ) + ) + + +@view_config( + route_name="edit_scheduled_action", + request_method="GET", + renderer="edit_scheduled_action.html.j2", +) +@editor_required +def edit_scheduled_action_get_view(request): + namespace = request.namespace + namespace_dbsession = request.namespace_dbsession + media_short_id = request.matchdict.get("media_short_id") + action_id = request.matchdict.get("action_id") + + # Verify media exists + media = namespace_dbsession.query(Media).filter_by(short_id=media_short_id).first() + if not media: + return Response("Media not found.", status=404) + + # Check if scheduled_actions table exists + try: + inspector = inspect(namespace_dbsession.bind) + if "scheduled_actions" not in inspector.get_table_names(): + return Response("Scheduled actions not supported.", status=404) + except: + return Response("Scheduled actions not supported.", status=404) + + # Find the scheduled action + scheduled_action = ( + namespace_dbsession.query(ScheduledAction) + .filter(ScheduledAction.id == action_id, ScheduledAction.media_id == media.id) + .first() + ) + + if not scheduled_action: + return Response("Scheduled action not found.", status=404) + + return { + "request": request, + "namespace": namespace, + "media": media, + "scheduled_action": scheduled_action, + } + + +@view_config(route_name="edit_scheduled_action", request_method="POST") +@editor_required +def edit_scheduled_action_post_view(request): + namespace = request.namespace + namespace_dbsession = request.namespace_dbsession + media_short_id = request.matchdict.get("media_short_id") + action_id = request.matchdict.get("action_id") + + # Verify media exists + media = namespace_dbsession.query(Media).filter_by(short_id=media_short_id).first() + if not media: + return Response("Media not found.", status=404) + + # Check if scheduled_actions table exists + try: + inspector = inspect(namespace_dbsession.bind) + if "scheduled_actions" not in inspector.get_table_names(): + return Response("Scheduled actions not supported.", status=404) + except: + return Response("Scheduled actions not supported.", status=404) + + # Find the scheduled action + scheduled_action = ( + namespace_dbsession.query(ScheduledAction) + .filter(ScheduledAction.id == action_id, ScheduledAction.media_id == media.id) + .first() + ) + + if not scheduled_action: + return Response("Scheduled action not found.", status=404) + + # Update the scheduled action + new_action_type = request.POST.get("action_type") + new_scheduled_date = request.POST.get("scheduled_date") + + if new_action_type not in ["set_public", "set_private", "set_unlisted", "delete"]: + return Response("Invalid action type.", status=400) + + try: + scheduled_date = datetime.datetime.fromisoformat(new_scheduled_date) + except ValueError: + return Response("Invalid date format.", status=400) + + scheduled_action.action_type = new_action_type + scheduled_action.scheduled_date = scheduled_date + namespace_dbsession.flush() + + request.session.flash( + f"Scheduled action updated to {new_action_type.replace('set_', '').replace('_', ' ')} on {scheduled_date.strftime('%Y-%m-%d %H:%M')}." + ) + + return HTTPFound( + location=request.route_url( + "view_media_details", + namespace_short_id=namespace.short_id, + media_short_id=media.short_id, + ) + ) + + +@view_config( + route_name="add_scheduled_action", + request_method="GET", + renderer="add_scheduled_action.html.j2", +) +@editor_required +def add_scheduled_action_get_view(request): + namespace = request.namespace + namespace_dbsession = request.namespace_dbsession + media_short_id = request.matchdict.get("media_short_id") + + # Verify media exists + media = namespace_dbsession.query(Media).filter_by(short_id=media_short_id).first() + if not media: + return Response("Media not found.", status=404) + + # Check if scheduled_actions table exists + try: + inspector = inspect(namespace_dbsession.bind) + if "scheduled_actions" not in inspector.get_table_names(): + return Response("Scheduled actions not supported.", status=404) + except: + return Response("Scheduled actions not supported.", status=404) + + return { + "request": request, + "namespace": namespace, + "media": media, + } + + +@view_config(route_name="add_scheduled_action", request_method="POST") +@editor_required +def add_scheduled_action_post_view(request): + namespace = request.namespace + namespace_dbsession = request.namespace_dbsession + media_short_id = request.matchdict.get("media_short_id") + + # Verify media exists + media = namespace_dbsession.query(Media).filter_by(short_id=media_short_id).first() + if not media: + return Response("Media not found.", status=404) + + # Check if scheduled_actions table exists + try: + inspector = inspect(namespace_dbsession.bind) + if "scheduled_actions" not in inspector.get_table_names(): + return Response("Scheduled actions not supported.", status=404) + except: + return Response("Scheduled actions not supported.", status=404) + + # Get form data + action_type = request.POST.get("action_type") + scheduled_date_str = request.POST.get("scheduled_date") + + if action_type not in ["set_public", "set_private", "set_unlisted", "delete"]: + return Response("Invalid action type.", status=400) + + try: + scheduled_date = datetime.datetime.fromisoformat(scheduled_date_str) + except ValueError: + return Response("Invalid date format.", status=400) + + # Create new scheduled action + action_uuid = uuid.uuid4() + scheduled_action = ScheduledAction( + id=str(action_uuid), + media_id=media.id, + action_type=action_type, + scheduled_date=scheduled_date, + ) + namespace_dbsession.add(scheduled_action) + namespace_dbsession.flush() + + request.session.flash( + f"Scheduled {action_type.replace('set_', '').replace('_', ' ')} action added for {scheduled_date.strftime('%Y-%m-%d %H:%M')}." + ) + + return HTTPFound( + location=request.route_url( + "view_media_details", + namespace_short_id=namespace.short_id, + media_short_id=media.short_id, + ) + ) + + ################################################################################ # Main ################################################################################ @@ -1471,6 +2142,9 @@ def main(*config, **settings): session_factory_ = sessionmaker(bind=engine) Base.metadata.bind = engine + # Create tables if they don't exist + Base.metadata.create_all(engine) + DBSession = scoped_session(session_factory_) # Register with zope.sqlalchemy register(DBSession) @@ -1539,10 +2213,40 @@ def main(*config, **settings): "/namespace/{namespace_short_id}/media/{media_short_id}", ) + # Scheduled Actions Management + config.add_route( + "delete_scheduled_action", + "/namespace/{namespace_short_id}/media/{media_short_id}/scheduled_action/{action_id}/delete", + ) + config.add_route( + "edit_scheduled_action", + "/namespace/{namespace_short_id}/media/{media_short_id}/scheduled_action/{action_id}/edit", + ) + config.add_route( + "add_scheduled_action", + "/namespace/{namespace_short_id}/media/{media_short_id}/add_scheduled_action", + ) + config.scan() + + # Start the scheduler + start_scheduler() + return config.make_wsgi_app() +# Signal handler for graceful shutdown +def signal_handler(signum, frame): + log.info("Received shutdown signal, stopping scheduler...") + stop_scheduler() + sys.exit(0) + + +# Register signal handlers +signal.signal(signal.SIGINT, signal_handler) +signal.signal(signal.SIGTERM, signal_handler) + + # Expose the WSGI application callable for uwsgi uwsgi_app = main({}) diff --git a/migration_scripts/01_visibility_column.py b/migration_scripts/01_visibility_column.py new file mode 100644 index 0000000..c33a494 --- /dev/null +++ b/migration_scripts/01_visibility_column.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +""" +PyraFiles Migration Script: is_public to visibility column +========================================================= + +This script migrates existing PyraFiles databases from the old is_public boolean column +to the new visibility string column with three states: 'public', 'private', 'unlisted'. + +Migration logic: +- is_public = True → visibility = 'public' +- is_public = False → visibility = 'private' +- All is_public values are then set to FALSE (for compatibility) + +The is_public column is kept in the database but set to a consistent value. + +Usage: + python migrate_visibility.py [--data-dir /path/to/data] [--dry-run] + +Options: + --data-dir Path to PyraFiles data directory (default: ./data) + --dry-run Show what would be migrated without making changes + --help Show this help message +""" + +import os +import sys +import argparse +import sqlite3 +import glob +import logging +from pathlib import Path + +# Set up logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) +log = logging.getLogger(__name__) + + +def check_table_structure(cursor, table_name): + """Check if table exists and get its column information.""" + cursor.execute(f"PRAGMA table_info({table_name})") + columns = cursor.fetchall() + column_names = [col[1] for col in columns] + return columns, column_names + + +def migrate_database(db_path, dry_run=False): + """Migrate a single database file.""" + log.info(f"Processing database: {db_path}") + + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + # Check if media table exists + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='media'") + if not cursor.fetchone(): + log.info(f" No media table found in {db_path}, skipping") + conn.close() + return True + + # Get current table structure + columns, column_names = check_table_structure(cursor, 'media') + + has_is_public = 'is_public' in column_names + has_visibility = 'visibility' in column_names + + log.info(f" Table structure: has_is_public={has_is_public}, has_visibility={has_visibility}") + + if not has_is_public and has_visibility: + log.info(f" Database already migrated, skipping") + conn.close() + return True + + if not has_is_public: + log.warning(f" No is_public column found in {db_path}, skipping") + conn.close() + return True + + # Get count of records to migrate + cursor.execute("SELECT COUNT(*) FROM media") + total_records = cursor.fetchone()[0] + + if total_records == 0: + log.info(f" No records to migrate in {db_path}") + conn.close() + return True + + log.info(f" Found {total_records} records to migrate") + + if dry_run: + # Show what would be migrated + cursor.execute("SELECT id, filename, is_public FROM media LIMIT 10") + sample_records = cursor.fetchall() + log.info(f" Sample records that would be migrated:") + for record in sample_records: + id_val, filename, is_public = record + new_visibility = 'public' if is_public else 'private' + log.info(f" {filename}: is_public={is_public} → visibility='{new_visibility}'") + if total_records > 10: + log.info(f" ... and {total_records - 10} more records") + + conn.close() + return True + + # Perform the actual migration + log.info(f" Starting migration...") + + # Step 1: Add visibility column if it doesn't exist + if not has_visibility: + log.info(f" Adding visibility column...") + cursor.execute("ALTER TABLE media ADD COLUMN visibility TEXT DEFAULT 'public'") + + # Step 2: Migrate data + log.info(f" Migrating data...") + cursor.execute(""" + UPDATE media + SET visibility = CASE + WHEN is_public = 1 THEN 'public' + WHEN is_public = 0 THEN 'private' + ELSE 'public' + END + """) + + migrated_count = cursor.rowcount + log.info(f" Updated {migrated_count} records") + + # Step 3: Verify migration + cursor.execute("SELECT visibility, COUNT(*) FROM media GROUP BY visibility") + visibility_counts = cursor.fetchall() + log.info(f" Post-migration visibility distribution:") + for visibility, count in visibility_counts: + log.info(f" {visibility}: {count} records") + + # Step 4: Set is_public to FALSE (0) for all records + # We'll keep the column but just set it to a consistent state + log.info(f" Setting is_public to FALSE for all records...") + cursor.execute("UPDATE media SET is_public = 0") + + # Commit changes + conn.commit() + log.info(f" ✅ Migration completed successfully!") + + except Exception as e: + log.error(f" ❌ Error migrating {db_path}: {e}") + if 'conn' in locals(): + conn.rollback() + return False + finally: + if 'conn' in locals(): + conn.close() + + return True + + +def find_namespace_databases(data_dir): + """Find all namespace database files.""" + pattern = os.path.join(data_dir, "namespace_*.db") + namespace_dbs = glob.glob(pattern) + return namespace_dbs + + +def main(): + parser = argparse.ArgumentParser( + description='Migrate PyraFiles databases from is_public to visibility column', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__ + ) + parser.add_argument( + '--data-dir', + default='./data', + help='Path to PyraFiles data directory (default: ./data)' + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='Show what would be migrated without making changes' + ) + + args = parser.parse_args() + + data_dir = Path(args.data_dir) + + if not data_dir.exists(): + log.error(f"Data directory does not exist: {data_dir}") + sys.exit(1) + + if not data_dir.is_dir(): + log.error(f"Data directory is not a directory: {data_dir}") + sys.exit(1) + + log.info(f"Starting PyraFiles migration...") + log.info(f"Data directory: {data_dir.absolute()}") + log.info(f"Dry run: {args.dry_run}") + + # Find all namespace databases + namespace_dbs = find_namespace_databases(str(data_dir)) + + if not namespace_dbs: + log.info("No namespace databases found. Nothing to migrate.") + sys.exit(0) + + log.info(f"Found {len(namespace_dbs)} namespace databases to check:") + for db_path in sorted(namespace_dbs): + log.info(f" {os.path.basename(db_path)}") + + if args.dry_run: + log.info("\n" + "="*50) + log.info("DRY RUN MODE - No changes will be made") + log.info("="*50) + + # Migrate each database + success_count = 0 + error_count = 0 + + for db_path in sorted(namespace_dbs): + if migrate_database(db_path, dry_run=args.dry_run): + success_count += 1 + else: + error_count += 1 + + # Summary + log.info("\n" + "="*50) + log.info("MIGRATION SUMMARY") + log.info("="*50) + log.info(f"Total databases: {len(namespace_dbs)}") + log.info(f"Successfully processed: {success_count}") + log.info(f"Errors: {error_count}") + + if args.dry_run: + log.info("\nThis was a dry run. No changes were made.") + log.info("Run without --dry-run to perform the actual migration.") + elif error_count == 0: + log.info("\n✅ All databases migrated successfully!") + else: + log.warning(f"\n⚠️ {error_count} databases had errors. Check logs above.") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/templates/add_scheduled_action.html.j2 b/templates/add_scheduled_action.html.j2 new file mode 100644 index 0000000..4f938fe --- /dev/null +++ b/templates/add_scheduled_action.html.j2 @@ -0,0 +1,28 @@ + +{% extends 'base.html.j2' %} + +{% block title %}Add Scheduled Action{% endblock %} + +{% block content %} +

Add Scheduled Action for "{{ media.title or media.filename }}"

+ +
+ + + +

+ + +

+ + +
+ +

Back to Media Details

+{% endblock %} diff --git a/templates/base.html.j2 b/templates/base.html.j2 index dc2ae5f..e9d9edb 100644 --- a/templates/base.html.j2 +++ b/templates/base.html.j2 @@ -1,3 +1,4 @@ + @@ -127,6 +128,9 @@
  • Home
    • {% if request.user and request.user.is_verified %}
  • Create Namespace
    • + {% if request.namespace %} +
  • Manage Namespace
    • + {% endif %}
  • Profile
  • @@ -149,6 +153,9 @@
  • Home
    • {% if request.user and request.user.is_verified %}
  • Create Namespace
    • + {% if request.namespace %} +
  • Manage Namespace
    • + {% endif %}
  • Profile
  • diff --git a/templates/edit_media.html.j2 b/templates/edit_media.html.j2 index d0a82fc..39ed9dc 100644 --- a/templates/edit_media.html.j2 +++ b/templates/edit_media.html.j2 @@ -1,3 +1,4 @@ + {% extends 'base.html.j2' %} {% block title %}Edit Media{% endblock %} @@ -6,16 +7,34 @@

    Edit Media

    +

    - -

    + {% if ascii_content %} + +

    + {% endif %} -

    + +

    + +
    + Visibility +
    +
    + +

    • diff --git a/templates/edit_scheduled_action.html.j2 b/templates/edit_scheduled_action.html.j2 new file mode 100644 index 0000000..7ad694e --- /dev/null +++ b/templates/edit_scheduled_action.html.j2 @@ -0,0 +1,30 @@ + + +{% extends 'base.html.j2' %} + +{% block title %}Edit Scheduled Action{% endblock %} + +{% block content %} +

    Edit Scheduled Action for "{{ media.title or media.filename }}"

    + +
    + + + +

    + + +

    + + +
    + +

    Back to Media Details

    +{% endblock %} diff --git a/templates/home.html.j2 b/templates/home.html.j2 index 294d783..8fca32f 100644 --- a/templates/home.html.j2 +++ b/templates/home.html.j2 @@ -1,3 +1,4 @@ + {% extends 'base.html.j2' %} {% block title %}Home{% endblock %} @@ -11,37 +12,122 @@ Welcome, Guest! {% endif %}

    -

    This application allows environments and agents to centrally manage and share files within namespaces. You can create namespaces to organize files and control access.

    + +
    +

    🚀 What is PyraFiles?

    +

    PyraFiles is a powerful, multi-tenant file sharing and management platform designed for teams, organizations, and automated systems. Built with security and flexibility in mind, it provides a comprehensive solution for managing digital assets across different environments.

    +
    {% if request.user and request.user.is_verified %} -

    Your Namespaces

    - {% if user_namespaces %} +
    +

    Your Namespaces

    + {% if user_namespaces %} +
      + {% for ns in user_namespaces %} +
    • + {{ ns.name }} ({{ ns.role }}) + {% if ns.role == "owner" %} + | Manage + {% endif %} +
      • + {% endfor %} +
    + {% else %} +

    You are not a member of any namespaces. Create your first namespace to get started!

    + {% endif %} +
    +{% endif %} + +
    +

    Public Namespaces

    + {% if public_namespaces %}
      - {% for ns in user_namespaces %} + {% for ns in public_namespaces %}
    • - {{ ns.name }} ({{ ns.role }}) - {% if ns.role == "owner" %} - | Manage - {% endif %} + {{ ns.name }}
      • {% endfor %}
    {% else %} -

    You are not a member of any namespaces.

    +

    No public namespaces available.

    {% endif %} +
    + +{% if not request.user or not request.user.is_verified %} +
    +

    Get Started

    +

    Login or create an account to start managing your files with PyraFiles!

    +
    {% endif %} -

    Public Namespaces

    -{% if public_namespaces %} +
    +

    ✨ Key Features

    + +

    🔐 Advanced Authentication & Authorization

      - {% for ns in public_namespaces %} -
    • - {{ ns.name }} -
      • - {% endfor %} +
    • Email-based authentication with secure 6-digit verification codes
      • +
    • Dual access modes: Human users via web interface and automated agents via JWT tokens
      • +
    • Role-based permissions: Owner, Editor, and Reader roles with granular access control
      • +
    • Namespace isolation: Complete separation of data between different organizations
    -{% else %} -

    No public namespaces available.

    -{% endif %} + +

    📁 Flexible File Management

    +
      +
    • Multi-format support: Images, audio, video, documents, and text files up to 310MB
      • +
    • Three visibility modes: +
        +
      • Public: Visible in listings and accessible to all
        • +
      • Private: Only accessible to namespace members
        • +
      • Unlisted: Accessible via direct link but hidden from listings
        • +
      +
      • +
    • Built-in text editor: Edit ASCII files (JSON, TXT, YAML, etc.) directly in the browser
      • +
    • File previews: View first 100 lines of text files inline
      • +
    + +

    ⏰ Scheduled Actions

    +
      +
    • Automated visibility changes: Schedule files to become public, private, or unlisted
      • +
    • Automated deletion: Set expiration dates for temporary files
      • +
    • Background processing: Reliable scheduler ensures actions execute on time
      • +
    • Audit trail: Track all scheduled actions and their status
      • +
    + +

    🤖 Agent & API Support

    +
      +
    • JWT authentication: Secure token-based access for automated systems
      • +
    • Agent management: Create, revoke, and manage API access per namespace
      • +
    • RESTful API: Full programmatic access to all file operations
      • +
    • CORS support: Cross-origin requests enabled for web applications
      • +
    + +

    👥 Team Collaboration

    +
      +
    • User invitations: Invite team members via email with specific roles
      • +
    • Namespace sharing: Control who can view, edit, or manage your files
      • +
    • Public namespaces: Share collections openly while maintaining private spaces
      • +
    • Gravatar integration: Personalized user profiles
      • +
    + +

    🛡️ Security & Reliability

    +
      +
    • Secure file storage: Base64 encoding with SQLite backend
      • +
    • CSRF protection: Built-in protection against cross-site request forgery
      • +
    • Session management: Secure, long-lived sessions with proper timeouts
      • +
    • Input validation: Comprehensive validation of all user inputs
      • +
    • Error handling: Graceful error handling with user-friendly messages
      • +
    +
    + +
    +

    🎯 Perfect For

    +
      +
    • Development Teams: Share code snippets, documentation, and assets
      • +
    • Content Creators: Manage media files with scheduled publishing
      • +
    • Automated Systems: Integrate file storage into your applications via API
      • +
    • Organizations: Secure file sharing with proper access controls
      • +
    • Temporary Sharing: Share files with automatic expiration
      • +
    +
    {% endblock %} diff --git a/templates/upload_media.html.j2 b/templates/upload_media.html.j2 index d676d2b..36136f9 100644 --- a/templates/upload_media.html.j2 +++ b/templates/upload_media.html.j2 @@ -1,3 +1,4 @@ + {% extends 'base.html.j2' %} {% block title %}Upload Media{% endblock %} @@ -6,16 +7,43 @@

    Upload Media to Namespace '{{ namespace.name }}'

    + -

    +



    -

    +
    + Visibility +
    +
    + +

    + +
    + Scheduled Actions (Optional) + +

    + + +

    + + +

    + + +

    +
    diff --git a/templates/view_media_details.html.j2 b/templates/view_media_details.html.j2 index 828ef29..da35050 100644 --- a/templates/view_media_details.html.j2 +++ b/templates/view_media_details.html.j2 @@ -1,3 +1,4 @@ + {% extends 'base.html.j2' %} {% block title %}Media Details{% endblock %} @@ -9,7 +10,39 @@

    Type: {{ media.media_type }}

    Size: {{ media.size|filesizeformat }}

    Uploaded: {{ media.upload_date.strftime('%Y-%m-%d %H:%M:%S') }}

    -

    Public: {{ 'Yes' if media.is_public else 'No' }}

    +

    Visibility: {{ media.visibility|title }}

    + +{% if scheduled_actions %} +

    Scheduled Actions

    +
      + {% for action in scheduled_actions %} +
    • + {{ action.action_type.replace('set_', '').replace('_', ' ')|title }} - {{ action.scheduled_date.strftime('%Y-%m-%d %H:%M:%S') }} ({{ action.status }}) + {% if is_owner_or_editor %} + | Edit + |
      + + +
      + {% endif %} +
      • + {% endfor %} +
    +{% if is_owner_or_editor %} +

    Add Scheduled Action

    +{% endif %} +{% else %} +{% if is_owner_or_editor %} +

    Scheduled Actions

    +

    No scheduled actions for this media.

    +

    Add Scheduled Action

    +{% endif %} +{% endif %} + +{% if ascii_preview %} +

    Text Preview (First 100 lines)

    +
    {{ ascii_preview }}
    +{% endif %}

    Preview

    {% if media.media_type == 'image' %} @@ -30,7 +63,8 @@

    Actions

    - Download + View + | Download {% if is_owner_or_editor %} | Edit |