From 596dbb923d152184516da9ae1c447f0dc59d4e4b Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Mon, 6 Jan 2025 19:48:23 -0500 Subject: [PATCH] A test_agent.sh & a README.rst deleted: .app.py.swp modified: .gitignore new file: README.rst modified: app.py new file: initialize_db.py new file: test_agent.sh --- .app.py.swp | Bin 16384 -> 0 bytes .gitignore | 4 + README.rst | 303 +++++++++++++++++++++++++++++++++++++++++++++++ app.py | 177 ++++++++++++++------------- initialize_db.py | 68 +++++++++++ test_agent.sh | 75 ++++++++++++ 6 files changed, 547 insertions(+), 80 deletions(-) delete mode 100644 .app.py.swp create mode 100644 README.rst create mode 100644 initialize_db.py create mode 100644 test_agent.sh diff --git a/.app.py.swp b/.app.py.swp deleted file mode 100644 index 010a93d88787638c73c8d7698132a5efbc1beadc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHOTZ|<|8E!;mSuPq)eBqL^$C#cl+q0}J!jOaP>;*=5$htFQg1c*{&*?hn)NXfI z)0de!J6UdGhz}+jV?>fwqa;4)5~HFf;+w{(i4PEkNc6=Rd=VE>yaIlIUHV*R212qA znsk$Ir@QK}zyA8_uj=~$KD2j#bYYR*;@{xldWYkD<*!BWvQJ-q(WCQ@)0OK~`i56K z4%1Vi5AnXt{FIm5w73@%vprGHR#_~=$gk3}%Da9?CbMCd`PpEbc1>}^K*PX=VBkvU z{d;cMO`4|M_po=pbLB!PZFDpYGz>HhGz>HhGz>HhGz>HhGz|QoFi>VYo%>+xTa3M@ z&EJbA{=R0e*pzQf)W2%#6I1^DMExtKzA)v>rUTp0e@uPf)cW7PhRdg-vB-a+zosHxC!{fPRDr)cmTK`xDSYdWndP#6!`mP zj`K3`H{h4RH-QAW4!9P$26$-)`UV~XI>2qfb--UPb(|N0XMrCAp94+<10Vwq0k6Hy zab5tP1|9>x27DFx1R#J#U<&xvC64nQ;M>5LfiD4PfqQ|yzz*Prw>r+BfZqb=fE);c z1>jcTJaQ600e%d81`xpAz{S9`nBRv10o(?(0Gjiwf$c7aYl_{$Q=W$8my$^o{F=o;CUJ*oONZDMvfH|U6#s{@pVy@BJBkvHdoh< z6|)vAe5lF?-U_6q{jP{ho(JuLXO*{l6)#Gqg!&?pyePftTH$y-G`cmWA)}qodTx=R!Y|6r7}gF;TGgkF&`o-wyG*JaQwFE_NLy;0S{f*R{S|9y*HHms|UDrLThe`Z&syydusprcJ zE0m?B=wPZMCUZSm(@a@tbwYs?@h*)aFwK=8d&anF*sIJkY_8?_krdB|vu(;(1z!ug zRobZ=s#eTgM(WPxM_+M&}d@W^Ld`j z-0Qjr=`WT=8evT`o`fQvV;lU8v6ruRcviB5>JJtU3kx>lq_nMw(QTr@T1g|);<>5J zc@__D=j&mT#hiYh=FYHVr3_RzCuUeu&^8e0b?=$QP8O1H6-k-JqU|p&E+6sSSa!m= zFN+c?_UzSUmifT=A`~5Sm!*3G?W}%>6sd|saA;MvrAb4%rK)|BcS9u7vnAYw?Oh&w2^*pz=?PaJO%68p(sF*vRZjdTV^=Q4Z1T$gm*h*+wqTY^n zvw*dT9i-`COSG(vjmDxW22W-sb~u{*tjaTqMK!}R?9X~XP!q*EGA*e!{93J1nHDnU z%HL?V^>ljHk{M=fo?(iqiRIUI`(>FGbF;H+YioWl4_C1z=K9Br>UMUc9~S+fBO`w1 zgGFnj5U2T>_eCv%1&6Ibrut1&3t?*Lo?In~_Ik=>pQqFwdrjHhb+bD??f3aQ-YXcY zd8g@-wY10kpxe=wMIlhTXgCC1rMo6276_g>GO}&V3N! zjWIv1*q1z?}Xg(;(C(VwUrRH8ndp5tDfi%Jf)5(p1Q!7GsW)YF*Zte z2zz3QwO?HLJ{&kbsKUQJIuc+DKa5*UK@iX}BnYt1F7UXfFt8=O?yAgod*OR<)C=mg z%-{m|#9`d&^JKt8lF@lD=RF+Rz$?xT3%Y1R(lL@H4o*d=xE~Q7Elt7{FP8)h1HUq+ z*LF6@!$d^ENs*om_t-8MKBOKGrmG^CsS+QR_W3F_6jmBuTXb^Krg!9+7~aeg&v{-; z>Vg0(Kj%dT58qZogO6r4N5}v{^UPmtCojAm?T&(hd4rY#Cn+q7}`ixr63)uYNlKNpD2J$e~4+n58 zSr!S`PfO}ptMB*AxJrzaK4}I#g^NRU!WKI+Pgj`vvnNhlyCixkkM>J>O7JB;vO{nT znGeRB(blS8*|c75Ykg)x@G!&1jxXbF?G)NpN9~l3`2d|Ltb{sUU_rr}>PQab7c1z>%_7&@xXR|NZ$CHIj?LD11t^ZcAGi%@ z0ha(zAa4II@O9vmz(c?QI1F41ynvYfN5GQ+#q_rTFCs=i5Bval5I6$7g82M-;0wTK z0gBn5M?C&T;1S?{U>|Th-~(3xmjgS2KZA?k0p|eXZoOk!YB%I zS(S+2BDt2*|Kk?e#2H0Mvy>N_9agpLcoruzm0n{9OuOy$s-sK7BM~Zv>TY&JC9YIf z7HO_Cvm2^7s0&OtTUg0sUh?40>YGK55$(2B3-ii1uasIDm3pShFWljPgNT<%2o!H( z;4oA-*?N|?F4cLyNlu93Nuu3p3U3yX4viQO^H`WP*Jh$)L{6I+p~7uB)c=d*rZ&3R zv_jwH32YZT%4JR=PQ;Y|6>LqEeU@M)u_EIg(G?x0GUMC%l|Hzt{3Bwb4zy{9rM%$m z7L|I2HV>4q({kOcha7*4I<003c{&wh(#M4b9a^q{Zw!kpVctHKo|I`kz{iA~X?wO^ zds!~=p@0upoGsYSXlyB1wL^LHyq)+e&ym3xiks){#Gy^3LuvhJ2fh!)Y&R!;ndm{( z(|wdS7yN=23xkqdh+llQ=je1loTco`$fF{)Bh=Y#>^0+ zwV}kQy>G-4(z0>bj(KY`|31Sgh0{`uO&F`Y-Dh~B-k_vr$k$!kJc(BiMQsyDZTdtl z&B@FOX|Cn*x*AUycqVZWban7-XAFx3UxLPEwGs1(^L+S2NbeuNEwHeVGgP6xLw!vs zq#j2S4<0pWX>r|`{+C)Fm=9Kt9-blpOObRA!n}uMr;Aik=TuNo@~iCQ8&%}au|pxg z44t7jxQvmlD2modwuE%)9?yor%&z!^ah zCh%z^VyoB?Dch@ZLzL1rIUyFdu~BdG3Y~C5W+u!#{U8g=zUP)4d4B}X^s3Rm;Gf*H zd8al_ZFb4SS3@gR8*j|C8o@I4gHJ11_Ii}$O@6;jVzkCrQ6#MCL!Vl06OG5$n@Brx aMX6$&Ix7?XK3dQfbxN&Fl(KEoZ~q13OiY9T diff --git a/.gitignore b/.gitignore index 98e6ef6..63ad4cf 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ *.db +*.swp +env/ +__pycache__/ +cookies.txt diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..1cac10c --- /dev/null +++ b/README.rst @@ -0,0 +1,303 @@ +=========================================== +pyrafiles - A Sophisticated Pyramid Project +=========================================== + +``pyrafiles`` is a **public domain** application built on the `Pyramid `_ framework. It demonstrates a flexible, multi-database design where each verified user maintains their own SQLite database for media uploads (images, audio, video). The main database (``main.db``) stores system- and user-level data, while each user’s personal DB file handles their specific uploads. + +Key Features +============ + +- **Passwordless Login** using email verification codes. +- **Guest Sessions** for unverified visitors. +- **Per-user SQLite** databases for media uploads (limit ~30MB each). +- **Public/Private** media visibility controls. +- **Admin Tools** (import user records, manage site settings). +- **Agent-friendly** architecture: easily scriptable endpoints for login, verification, media uploads, etc. +- **Configurable** via environment variables (including secret keys). + +Git Repository +============== + +The project is maintained at: + +- `pyrafiles Git Repo `_ + +Since this is public domain, you can adapt and redistribute it freely. + + +Configuration via Environment +============================= + +`pyrafiles` fetches settings from environment variables with sensible defaults: + +- ``PYRAFILES_SECRET`` + The secret key for session signing. + If missing, ``pyrafiles`` automatically generates a **random 64-character** string at runtime & log all users out. + +- ``PYRAFILES_DB_URL`` + Connection string for the main database. Default: ``sqlite:///main.db``. + +- ``PYRAFILES_HOST`` and ``PYRAFILES_PORT`` + The host and port to serve on. Defaults: ``0.0.0.0`` (host), ``6544`` (port). + +- ``PYRAFILES_SMTP_HOST`` and ``PYRAFILES_SMTP_PORT`` + SMTP server details. Defaults: ``localhost:25``. + +- Any other environment variables you wish to incorporate can be accessed in the code. + + +Local Setup +=========== + +1. **Clone the Project** + + .. code-block:: bash + + git clone https://git.unturf.com/engineering/unturf/upload.unturf.com.git + cd upload.unturf.com + +2. **Create a Virtual Environment** + + .. code-block:: bash + + python -m venv venv + source venv/bin/activate # Linux/Mac + # or + venv\Scripts\activate.bat # Windows + +3. **Install Dependencies** + + .. code-block:: bash + + pip install -r requirements.txt + +4. **Run** + + Create the database if this is the first time running the applicaiton. + + .. code-block:: bash + + python initialize_db.py + + Otherwise: + + .. code-block:: bash + + # Optionally set PYRAFILES_SECRET if you want a custom secret. + export PYRAFILES_SECRET="YOUR_OWN_LONG_RANDOM_STRING" + python main.py + + If ``PYRAFILES_SECRET`` is **not** set, the app automatically generates a 64-char secret at runtime & log all users out. + +5. **Access** + + Point your browser to `http://localhost:6544` or `http://:` according to your environment variables. + + +Example: Agent Workflow Script +============================== + +Below is a sample Bash script showing how an **agent** might: + +1. Start the session (to get a cookie). +2. Log in with an email. +3. Prompt the user for a verification code. +4. Complete the verification. +5. Upload a file. + +.. code-block:: bash + + #!/usr/bin/env bash + # + # Usage: + # MAX_COOKIE_AGE=1800 ./agent_upload.sh /path/to/somefile.jpg + # + # Description: + # Upload a file to the pyrafiles server as an agent, making it public. + # - If cookies.txt is newer than MAX_COOKIE_AGE seconds (default 31536000, ~1 year), + # we skip re-login. + # - Otherwise, we prompt for OTP again. + # + # Notes: + # 1. If not set, MAX_COOKIE_AGE defaults to 31536000 (one year). + # 2. We assume the server runs on http://localhost:6544. Adjust BASE_URL if needed. + + BASE_URL="${BASE_URL:-http://localhost:6544}" + EMAIL="${EMAIL:-agent@example.com}" + MEDIA_FILE="$1" + # Default to 1 year in seconds if not provided: + MAX_COOKIE_AGE="${MAX_COOKIE_AGE:-31536000}" + + if [[ -z "$MEDIA_FILE" ]]; then + echo "Usage: MAX_COOKIE_AGE= $0 /path/to/mediafile" + exit 1 + fi + + # Check if cookies.txt is 'fresh enough': + COOKIE_FRESH=0 + if [[ -f cookies.txt ]]; then + # 'stat -c %Y' returns the file's mod time on Linux; 'stat -f %m' on macOS/BSD + MOD_TIME=$(stat -c %Y cookies.txt 2>/dev/null || stat -f %m cookies.txt 2>/dev/null) + NOW=$(date +%s) + AGE=$((NOW - MOD_TIME)) + + if (( AGE < MAX_COOKIE_AGE )); then + COOKIE_FRESH=1 + fi + fi + + if [[ "$COOKIE_FRESH" -eq 1 ]]; then + echo "Reusing existing cookies (file age < $MAX_COOKIE_AGE seconds)." + echo "Skipping OTP prompt." + else + echo "Starting new session (cookie missing or stale)..." + curl -s -c cookies.txt -b cookies.txt "$BASE_URL/" >/dev/null + + echo "Logging in with email: $EMAIL" + curl -s -c cookies.txt -b cookies.txt \ + -X POST \ + -F "email=$EMAIL" \ + "$BASE_URL/auth/login" + + echo "" + echo "Check your console or email for the 6-digit verification code." + read -p "Enter the 6-digit code: " VERIFICATION_CODE + + echo "Verifying code..." + curl -s -c cookies.txt -b cookies.txt \ + -X POST \ + -F "code=$VERIFICATION_CODE" \ + "$BASE_URL/auth/verify" + + echo "Cookies refreshed." + fi + + echo "" + echo "Uploading media: $MEDIA_FILE (public)" + curl -s -c cookies.txt -b cookies.txt \ + -X POST \ + -F "media_file=@$MEDIA_FILE" \ + -F "is_public=on" \ + "$BASE_URL/media/upload" + + echo "" + echo "Upload complete. File is public." + + + +Dockerfile with Caddy + uWSGI +============================= + +Below is an example Dockerfile that: + +- Uses **uWSGI** to run ``pyrafiles``. +- Uses **Caddy** as a reverse proxy (and optionally HTTPS if configured). +- Defines a **volume** for databases (so they are stored on the host). + +.. code-block:: dockerfile + + ########################################################### + # Stage 1: Build the Python environment + ########################################################### + FROM python:3.9-slim AS builder + + WORKDIR /app + COPY . /app + + # Install Python dependencies + RUN pip install --no-cache-dir -r requirements.txt + + ########################################################### + # Stage 2: Final image with Caddy + uWSGI + Python + ########################################################### + FROM caddy:2-alpine + + # Install Python, pip, and uWSGI from Alpine + RUN apk add --no-cache python3 py3-pip uwsgi-python3 + + WORKDIR /app + + # Copy the app from the builder + COPY --from=builder /app /app + + # Copy Caddyfile for the reverse proxy + # (Assuming you have a Caddyfile in your repo root) + COPY Caddyfile /etc/caddy/Caddyfile + + # Copy uWSGI config if desired + # For example (assume you created uwsgi.ini in your repo): + # [uwsgi] + # module = main:app + # master = true + # processes = 4 + # socket = 127.0.0.1:8080 + # vacuum = true + # die-on-term = true + COPY uwsgi.ini /app/uwsgi.ini + + # Environment variables (optional overrides). + # If PYRAFILES_SECRET is empty, the app itself generates a random 64-char secret & logs all users out. + ENV PYRAFILES_SECRET="" + ENV PYRAFILES_DB_URL="sqlite:///data/main.db" + ENV PYRAFILES_HOST="0.0.0.0" + ENV PYRAFILES_PORT="6544" + + # Expose HTTP and HTTPS + EXPOSE 80 + EXPOSE 443 + + # Define volume so host can persist user DBs outside the container + # We'll store DB files in /data + VOLUME ["/data"] + + # Command starts uWSGI and then starts Caddy + # The app itself checks for PYRAFILES_SECRET and auto-generates one if missing. + CMD ["/bin/sh", "-c", "\ + uwsgi --ini /app/uwsgi.ini & \ + caddy run --config /etc/caddy/Caddyfile \ + "] + +.. note:: + + - We use ``/data`` as the volume. By default, the environment variable ``PYRAFILES_DB_URL`` is set to ``sqlite:///data/main.db``, so the main DB (and any user DB files) go inside ``/data``. + - For user DB files, your app can also interpret an environment variable (like ``PYRAFILES_DB_BASEPATH=/data``) if you want to make that path configurable in the code. + - Make sure to **mount** a volume at ``/data`` when you run the container, e.g.: + + .. code-block:: bash + + docker run -d \ + -p 80:80 -p 443:443 \ + -v /my/local/dbfolder:/data \ + --name pyrafiles \ + pyrafiles-image:latest + + - With that, any user database is stored in ``/my/local/dbfolder`` on the host. + + +Tips +==== + +- For **production**, you likely want to set up a real SMTP server or third-party service (e.g. Mailgun, ImprovMV) and configure it via environment variables (``PYRAFILES_SMTP_HOST``, etc.). +- Ensure you mount a volume for persistent SQLite files if you want to avoid data loss when containers are removed or replaced. +- Because the project is **public domain**, you can adapt it without restriction, removing features, adding custom logic, etc. + + +License and Public Domain +========================= + +This project is in the public domain. You are free to use, adapt, and redistribute it without attribution or additional licensing. + +If you find `pyrafiles` helpful, feel free to contribute back or share your enhancements! + + +Support and Contact +=================== + +- Issues: Please open tickets at the `git.unturf.com project page `_. +- For general inquiries, you can reach out to the maintainers directly. + +We hope `pyrafiles` helps you get up and running quickly with a flexible media-sharing and multi-DB infrastructure! + +If you register an account, let me know and I will bless you as an developer to contribute. + +Enjoy and happy building! diff --git a/app.py b/app.py index 527df79..d351dd0 100644 --- a/app.py +++ b/app.py @@ -1,3 +1,6 @@ +############################################################################### +# app.py - Full pyrafiles Application with Unicode-safe Content-Disposition +############################################################################### import os import base64 import datetime @@ -11,6 +14,7 @@ import smtplib import mimetypes import json import logging +import unicodedata from email.mime.text import MIMEText from pyramid.config import Configurator @@ -47,6 +51,31 @@ from zope.sqlalchemy import register # Import zope.sqlalchemy log = logging.getLogger(__name__) +################################################################################ +# Environment Variables and Defaults +################################################################################ + +# If PYRAFILES_SECRET is missing, generate a random 64-character secret. +pyrafiles_secret = os.environ.get("PYRAFILES_SECRET", "") +if not pyrafiles_secret: + pyrafiles_secret = "".join( + random.choices(string.ascii_letters + string.digits, k=64) + ) + log.info(f"Generated random PYRAFILES_SECRET: {pyrafiles_secret}") + +# Database URL can be overridden by environment variable +APP_DIR = os.path.dirname(os.path.abspath(__file__)) +default_main_db_url = f"sqlite:///{os.path.join(APP_DIR, 'main.db')}" +DB_URL = os.environ.get("PYRAFILES_DB_URL", default_main_db_url) + +# Host and port for the application +HOST = os.environ.get("PYRAFILES_HOST", "0.0.0.0") +PORT = int(os.environ.get("PYRAFILES_PORT", "6544")) + +# SMTP host/port +smtp_host = os.environ.get("PYRAFILES_SMTP_HOST", "localhost") +smtp_port = int(os.environ.get("PYRAFILES_SMTP_PORT", "25")) + ################################################################################ # Helper Functions ################################################################################ @@ -66,7 +95,6 @@ def get_gravatar_url(email, size=100): def send_email(to_email, subject, body): - # For testing purposes, print the email content to the console log.debug("======= Email Sent =======") log.debug(f"To: {to_email}") log.debug(f"Subject: {subject}") @@ -79,7 +107,7 @@ def send_email(to_email, subject, body): msg["To"] = to_email try: - s = smtplib.SMTP("localhost", 25) + s = smtplib.SMTP(smtp_host, smtp_port) s.sendmail("noreply@example.com", [to_email], msg.as_string()) s.quit() except Exception as e: @@ -121,12 +149,12 @@ def get_current_user(request): # Generate UUID and short ID user_uuid = uuid.uuid4() - user_id = str(user_uuid) + new_user_id = str(user_uuid) short_id = uuid_to_short_id(user_uuid) - # Create a new guest user (do not create user database) + # Create a new guest user guest_user = User( - id=user_id, + id=new_user_id, short_id=short_id, email=None, # Guests don't have an email username=guest_username, @@ -156,17 +184,15 @@ def uuid_to_short_id(u): def short_id_to_uuid(sid): """Decode the short ID back to UUID, trying different padding lengths.""" - # Try with 0 to 3 padding characters - for padding_length in range(5): + for padding_length in range(6): try: padded = sid + ("=" * padding_length) bytes_data = base64.urlsafe_b64decode(padded) if len(bytes_data) == 16: # UUID is 16 bytes return uuid.UUID(bytes=bytes_data) - except Exception as e: + except Exception: continue - # If we get here, none of the padding attempts worked log.error( f"Failed to convert short_id {sid} to UUID after trying all padding lengths" ) @@ -174,8 +200,7 @@ def short_id_to_uuid(sid): def get_user_db_url(user_id): - """Return the database URL for the user's SQLite database.""" - APP_DIR = os.path.dirname(os.path.abspath(__file__)) + """Return the database URL for the user's SQLite database, in the same directory.""" db_file = os.path.join(APP_DIR, f"user_{user_id}.db") return f"sqlite:///{db_file}" @@ -189,12 +214,24 @@ def filesizeformat(value): return f"{value:.2f} PB" +def sanitize_filename_for_http_header(filename): + """ + Ensure that the filename is safe for Waitress (Latin-1 headers). + Converts to ASCII, replacing or removing characters that won't encode. + """ + normalized = unicodedata.normalize("NFKD", filename) + ascii_bytes = normalized.encode("ascii", "ignore") # drop non-ASCII + safe = ascii_bytes.decode("ascii") + # Replace any remaining bad chars with underscores + # e.g. keep alphanumerics, dots, underscores, hyphens, etc. + safe = re.sub(r"[^A-Za-z0-9._-]+", "_", safe) + return safe or "download" + + ################################################################################ # Database Setup ################################################################################ -APP_DIR = os.path.dirname(os.path.abspath(__file__)) -DB_URL = f"sqlite:///{os.path.join(APP_DIR, 'main.db')}" log.debug(f"Using database URL: {DB_URL}") # For debugging Base = declarative_base() @@ -212,7 +249,6 @@ class User(Base): enable_gravatar = Column(Boolean, default=False) # Gravatar support is_admin = Column(Boolean, default=False) # Admin flag - # Indexes for faster lookup __table_args__ = ( Index("ix_users_id", "id"), Index("ix_users_short_id", "short_id"), @@ -235,7 +271,6 @@ class Media(Base): is_public = Column(Boolean, default=True) size = Column(Integer, nullable=False) # Size in bytes - # Indexes for faster lookup __table_args__ = ( Index("ix_media_id", "id"), Index("ix_media_short_id", "short_id"), @@ -281,7 +316,7 @@ def get_user_dbsession_by_user_id(user_id, request): register(user_dbsession) # Register with zope.sqlalchemy # Attach cleanup callbacks - def cleanup(request): + def cleanup(_request): user_dbsession.remove() user_engine.dispose() @@ -334,7 +369,7 @@ def login_post_view(request): is_verified=False, ) session.add(user) - session.flush() # Use flush instead of commit + session.flush() # Generate 6-digit code code_str = f"{random.randint(0,999999):06d}" @@ -407,7 +442,7 @@ def verify_post_view(request): user_db_url, connect_args={"check_same_thread": False}, poolclass=StaticPool ) Base.metadata.create_all(user_engine) # Create tables in user's database - user_engine.dispose() # Dispose the engine + user_engine.dispose() return HTTPFound(location=request.route_url("home")) @@ -432,7 +467,7 @@ def profile_get_view(request): # Initialize stats total_uploads = 0 - total_size = 0 # in bytes + total_size = 0 user_dbsession = request.user_dbsession if user_dbsession: @@ -484,10 +519,14 @@ def download_database_view(request): return Response("Database file not found.", status=404) with open(db_file, "rb") as f: data = f.read() + + # Safely build a filename + download_filename = sanitize_filename_for_http_header(f"user_{user.id}.db") + response = Response(body=data, content_type="application/octet-stream") - response.headers["Content-Disposition"] = ( - f'attachment; filename="user_{user.id}.db"' - ) + response.headers[ + "Content-Disposition" + ] = f'attachment; filename="{download_filename}"' return response @@ -515,8 +554,8 @@ def export_user_record_view(request): # Convert to JSON string user_json = json.dumps(user_record).encode("utf-8") - # Create response response = Response(body=user_json, content_type="application/json") + # Plain ASCII filename is safe response.headers["Content-Disposition"] = 'attachment; filename="user_record.json"' return response @@ -534,9 +573,6 @@ def import_user_record_get_view(request): @view_config(route_name="import_user_record", request_method="POST") @admin_required def import_user_record_post_view(request): - # Admin-only import process - - # Get the uploaded user record file user_record_file = request.POST.get("user_record_file") if ( user_record_file is None @@ -609,7 +645,7 @@ def upload_media_post_view(request): if len(raw_bytes) > max_size: return Response("File size exceeds the 30MB limit.", status=400) - file_size = len(raw_bytes) # Store the size in bytes + file_size = len(raw_bytes) # Determine media type based on MIME type filename = media_file.filename @@ -622,6 +658,10 @@ def upload_media_post_view(request): # Get title from form title = request.POST.get("title", "").strip() + # If user did not enter a title, default to the original filename + if not title: + title = filename + # Encode content to base64 encoded_str = base64.b64encode(raw_bytes).decode("utf-8") @@ -637,7 +677,7 @@ def upload_media_post_view(request): short_id=media_short_id, user_id=user.id, filename=filename, - title=title, + title=title, # now has a fallback of filename media_type=media_type, media_b64=encoded_str, is_public=is_public, @@ -645,7 +685,6 @@ def upload_media_post_view(request): ) user_dbsession.add(media) user_dbsession.flush() - # No need to commit; transaction manager will handle it return HTTPFound( location=request.route_url( @@ -658,21 +697,19 @@ def upload_media_post_view(request): @view_config(route_name="list_media", renderer="list_media.html.j2") def list_media_view(request): - # Aggregate public media from all users + # Aggregate public media from all verified users s = request.dbsession users = s.query(User).filter(User.is_verified == True).all() media_list = [] for user in users: - # Get user_dbsession user_dbsession = get_user_dbsession_by_user_id(user.id, request) if not user_dbsession: - continue # Skip users without database - + continue media_items = user_dbsession.query(Media).filter(Media.is_public == True).all() - for media in media_items: + for m in media_items: media_list.append( { - "media": media, + "media": m, "username": user.username, "user_short_id": user.short_id, } @@ -689,12 +726,10 @@ def list_media_view(request): @view_config(route_name="user_media", renderer="user_media.html.j2") def user_media_view(request): - # View uploads by a particular user user_short_id = request.matchdict.get("user_short_id") - log.debug(f"Looking up user with short_id: {user_short_id}") # Debug + log.debug(f"Looking up user with short_id: {user_short_id}") try: - # First convert short_id to UUID user_uuid = short_id_to_uuid(user_short_id) if not user_uuid: log.error(f"Could not convert short_id {user_short_id} to UUID") @@ -702,7 +737,7 @@ def user_media_view(request): s = request.dbsession user = s.query(User).filter_by(id=str(user_uuid)).first() - log.debug(f"User found: {user}") # Debug + log.debug(f"User found: {user}") if not user: return Response("User not found.", status=404) @@ -710,22 +745,18 @@ def user_media_view(request): viewer = request.user is_owner = viewer and viewer.id == user.id - # Get user_dbsession user_dbsession = get_user_dbsession_by_user_id(user.id, request) if not user_dbsession: return Response("User has no uploads.", status=404) if is_owner: - # Show all media (public and private) media_items = user_dbsession.query(Media).all() else: - # Show only public media media_items = ( user_dbsession.query(Media).filter(Media.is_public == True).all() ) - # Sort media by upload date (recent first) - media_items.sort(key=lambda media: media.upload_date, reverse=True) + media_items.sort(key=lambda m: m.upload_date, reverse=True) return { "request": request, @@ -735,7 +766,7 @@ def user_media_view(request): } except Exception as e: - log.exception(f"Error processing user_short_id {user_short_id}") + log.exception(f"Error processing user_short_id {user_short_id}: {e}") return Response("Error processing request.", status=500) @@ -752,12 +783,10 @@ def view_media_details_view(request): return Response("User not found.", status=404) user_id = user.id - # Get user_dbsession user_dbsession = get_user_dbsession_by_user_id(user_id, request) if not user_dbsession: return Response("User database not found.", status=404) - # Lookup media by short_id media = user_dbsession.query(Media).filter_by(short_id=media_short_id).first() if not media: return Response("Media not found.", status=404) @@ -789,16 +818,13 @@ def delete_media_view(request): if viewer.short_id != user_short_id: return Response("You are not authorized to delete this media.", status=403) - user_dbsession = request.user_dbsession # Assume this exists for verified users - + user_dbsession = request.user_dbsession media = user_dbsession.query(Media).filter_by(short_id=media_short_id).first() if not media: return Response("Media not found.", status=404) - # Delete the media user_dbsession.delete(media) user_dbsession.flush() - # No need to commit; transaction manager will handle it return HTTPFound( location=request.route_url("user_media", user_short_id=viewer.short_id) @@ -819,8 +845,7 @@ def edit_media_get_view(request): if viewer.short_id != user_short_id: return Response("You are not authorized to edit this media.", status=403) - user_dbsession = request.user_dbsession # Assume this exists for verified users - + user_dbsession = request.user_dbsession media = user_dbsession.query(Media).filter_by(short_id=media_short_id).first() if not media: return Response("Media not found.", status=404) @@ -843,8 +868,7 @@ def edit_media_post_view(request): if viewer.short_id != user_short_id: return Response("You are not authorized to edit this media.", status=403) - user_dbsession = request.user_dbsession # Assume this exists for verified users - + user_dbsession = request.user_dbsession media = user_dbsession.query(Media).filter_by(short_id=media_short_id).first() if not media: return Response("Media not found.", status=404) @@ -857,7 +881,7 @@ def edit_media_post_view(request): new_media_file = request.POST.get("media_file") if new_media_file and getattr(new_media_file, "filename", "").strip(): raw_bytes = new_media_file.file.read() - max_size = 30 * 1024 * 1024 # 30 MB + max_size = 30 * 1024 * 1024 if len(raw_bytes) > max_size: return Response("File size exceeds the 30MB limit.", status=400) file_size = len(raw_bytes) @@ -868,7 +892,6 @@ def edit_media_post_view(request): media_type = mime_type.split("/")[0] encoded_str = base64.b64encode(raw_bytes).decode("utf-8") - # Update media fields media.filename = filename media.media_type = media_type media.media_b64 = encoded_str @@ -879,7 +902,6 @@ def edit_media_post_view(request): media.is_public = is_public user_dbsession.flush() - # No need to commit; transaction manager will handle it return HTTPFound( location=request.route_url( @@ -908,40 +930,37 @@ def view_media_view(request): return Response("User not found.", status=404) user_id = user.id - # Get user_dbsession user_dbsession = get_user_dbsession_by_user_id(user_id, request) if not user_dbsession: return Response("User database not found.", status=404) - # Lookup media by short_id media = user_dbsession.query(Media).filter_by(short_id=media_short_id).first() if not media: return Response("Media not found.", status=404) viewer = request.user - is_owner = viewer and viewer.id == user.id + is_owner = viewer and viewer.id == user_id if not media.is_public and not is_owner: return Response("Media not available.", status=403) - # Decode base64 content media_data = base64.b64decode(media.media_b64) mime_type = get_mime_type(media.filename) - # Prepare filename for download + # Build a safe filename (avoid Unicode issues in the header) if media.title: - # Use title as filename, slugify it file_extension = os.path.splitext(media.filename)[1] - download_filename = f"{slugify(media.title)}{file_extension}" + raw_title = media.title + download_filename = sanitize_filename_for_http_header( + f"{raw_title}{file_extension}" + ) else: - # Use original filename - download_filename = media.filename + download_filename = sanitize_filename_for_http_header(media.filename) - # Check if the user wants to download the file + # Check if user wants attachment or inline download = request.GET.get("download", "false").lower() == "true" content_disposition = "attachment" if download else "inline" - # Serve the media content with appropriate headers response = Response(body=media_data, content_type=mime_type) response.headers.update( { @@ -962,9 +981,9 @@ def main(global_config=None, **settings): # Configure logging logging.basicConfig(level=logging.DEBUG) - # Set up the session factory with the specified settings + # Set up the session factory session_factory = SignedCookieSessionFactory( - secret="it-is-a-secret-you-must-change", + secret=pyrafiles_secret, hashalg="sha512", timeout=31104000, # Approx. one year in seconds max_age=31104000, # Set Max-Age attribute on cookie @@ -983,7 +1002,6 @@ def main(global_config=None, **settings): config.include("pyramid_tm") # Include pyramid_tm for transaction management # Add .html.j2 extension for Jinja2 templates - # Set up Jinja2 template search path config.add_jinja2_renderer(".j2") config.add_jinja2_search_path("templates", name=".j2") @@ -993,14 +1011,13 @@ def main(global_config=None, **settings): engine = create_engine( DB_URL, connect_args={"check_same_thread": False}, poolclass=StaticPool ) - session_factory = sessionmaker(bind=engine) + session_factory_ = sessionmaker(bind=engine) Base.metadata.bind = engine - # Use a scoped_session - DBSession = scoped_session(session_factory) + DBSession = scoped_session(session_factory_) register(DBSession) # Register with zope.sqlalchemy - # Add user to all requests. + # Add user to all requests config.add_request_method(callable=get_current_user, name="user", reify=True) # Provide dbsession to requests @@ -1025,10 +1042,10 @@ def main(global_config=None, **settings): config.add_route("export_user_record", "/auth/export_user_record") config.add_route("import_user_record", "/admin/import_user_record") - # Media - Reordered routes with most specific first + # Media config.add_route("upload_media", "/media/upload") config.add_route("list_media", "/media/list") - config.add_route("user_media", "/media/user/{user_short_id}") # Moved earlier + config.add_route("user_media", "/media/user/{user_short_id}") config.add_route( "view_media_details", "/media/{user_short_id}/{media_short_id}/details" ) @@ -1045,5 +1062,5 @@ def main(global_config=None, **settings): if __name__ == "__main__": app = main() - log.info("Serving on http://localhost:6544") - serve(app, host="0.0.0.0", port=6544) + log.info(f"Serving on http://{HOST}:{PORT}") + serve(app, host=HOST, port=PORT) diff --git a/initialize_db.py b/initialize_db.py new file mode 100644 index 0000000..0c45815 --- /dev/null +++ b/initialize_db.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +""" +Initialize or upgrade the main database for pyrafiles. + +Usage: + python initialize_db.py + +Environment Variables: + PYRAFILES_DB_URL (optional): + The SQLAlchemy database URL (e.g., "sqlite:///main.db"). + Defaults to "sqlite:///main.db" if not set. + +Description: + Creates or updates all tables referenced by `Base.metadata`. + If the database file/tables do not exist, they will be created. +""" + +import os +import sys +import logging +import transaction + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +# Adjust these imports to your actual project structure: +# For example, if your models and Base are defined in "models.py", do: +# from pyrafiles import Base +# If you keep Base in a separate module, adjust accordingly. +from app import Base # or from your_project.models import Base + +def usage(): + script = os.path.basename(sys.argv[0]) + print(f"Usage: {script}") + print("Example:") + print(f" python {script}") + sys.exit(1) + +def main(): + logging.basicConfig(level=logging.INFO) + if len(sys.argv) > 2: + # We only expect optional arguments. If needed, parse them here. + usage() + + # Read environment variable for DB URL + db_url = os.environ.get("PYRAFILES_DB_URL", "sqlite:///main.db") + logging.info(f"Using DB URL: {db_url}") + + # Set up engine + engine = create_engine( + db_url, + connect_args={"check_same_thread": False} if "sqlite" in db_url else {}, + poolclass=StaticPool if "sqlite" in db_url else None, + ) + + SessionFactory = sessionmaker(bind=engine) + + # Create or upgrade tables + with transaction.manager: + logging.info("Creating or upgrading tables using Base.metadata.create_all()") + Base.metadata.create_all(engine) + + logging.info("Database initialization complete.") + +if __name__ == "__main__": + main() + diff --git a/test_agent.sh b/test_agent.sh new file mode 100644 index 0000000..2973323 --- /dev/null +++ b/test_agent.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# +# Usage: +# MAX_COOKIE_AGE=1800 ./agent_upload.sh /path/to/somefile.jpg +# +# Description: +# Upload a file to the pyrafiles server as an agent, making it public. +# - If cookies.txt is newer than MAX_COOKIE_AGE seconds (default 31536000, ~1 year), +# we skip re-login. +# - Otherwise, we prompt for OTP again. +# +# Notes: +# 1. If not set, MAX_COOKIE_AGE defaults to 31536000 (one year). +# 2. We assume the server runs on http://localhost:6544. Adjust BASE_URL if needed. + +BASE_URL="${BASE_URL:-http://localhost:6544}" +EMAIL="${EMAIL:-agent@example.com}" +MEDIA_FILE="$1" +# Default to 1 year in seconds if not provided: +MAX_COOKIE_AGE="${MAX_COOKIE_AGE:-31536000}" + +if [[ -z "$MEDIA_FILE" ]]; then + echo "Usage: MAX_COOKIE_AGE= $0 /path/to/mediafile" + exit 1 +fi + +# Check if cookies.txt is 'fresh enough': +COOKIE_FRESH=0 +if [[ -f cookies.txt ]]; then + # 'stat -c %Y' returns the file's mod time on Linux; 'stat -f %m' on macOS/BSD + MOD_TIME=$(stat -c %Y cookies.txt 2>/dev/null || stat -f %m cookies.txt 2>/dev/null) + NOW=$(date +%s) + AGE=$((NOW - MOD_TIME)) + + if (( AGE < MAX_COOKIE_AGE )); then + COOKIE_FRESH=1 + fi +fi + +if [[ "$COOKIE_FRESH" -eq 1 ]]; then + echo "Reusing existing cookies (file age < $MAX_COOKIE_AGE seconds)." + echo "Skipping OTP prompt." +else + echo "Starting new session (cookie missing or stale)..." + curl -s -c cookies.txt -b cookies.txt "$BASE_URL/" >/dev/null + + echo "Logging in with email: $EMAIL" + curl -s -c cookies.txt -b cookies.txt \ + -X POST \ + -F "email=$EMAIL" \ + "$BASE_URL/auth/login" + + echo "" + echo "Check your console or email for the 6-digit verification code." + read -p "Enter the 6-digit code: " VERIFICATION_CODE + + echo "Verifying code..." + curl -s -c cookies.txt -b cookies.txt \ + -X POST \ + -F "code=$VERIFICATION_CODE" \ + "$BASE_URL/auth/verify" + + echo "Cookies refreshed." +fi + +echo "" +echo "Uploading media: $MEDIA_FILE (public)" +curl -s -c cookies.txt -b cookies.txt \ + -X POST \ + -F "media_file=@$MEDIA_FILE" \ + -F "is_public=on" \ + "$BASE_URL/media/upload" + +echo "" +echo "Upload complete. File is public."