Update .gitlab-ci.yml

.gitlab-ci.yml

@@ -1,30 +1,127 @@
 stages:
   - deploy
+  - deploy-streamlit
 
 variables:
-  # Change these to match your desired domain/app name
+  BACKEND_APP_NAME: "slop.unturf.com"
-  STREAMLIT_APP_NAME: "art.ai.unturf.com"
+  BACKEND_APP_DIR: "/opt/${BACKEND_APP_NAME}"
+  BACKEND_VENV_DIR: "${BACKEND_APP_DIR}/venv"
+  STREAMLIT_APP_NAME: "streamlit.slop.unturf.com"
   STREAMLIT_APP_DIR: "/opt/${STREAMLIT_APP_NAME}"
   STREAMLIT_VENV_DIR: "${STREAMLIT_APP_DIR}/venv"
-
-  # Adjust if your main Streamlit script has a different filename
-  STREAMLIT_SCRIPT_FILENAME: "streamlit_art_app.py"
-
-  # Caddy-related directories
   CADDYFILE_DIR: "/etc/caddy"
   CADDY_SNIPPETS_DIR: "/etc/caddy/caddy_snippets"
 
 deploy:
   stage: deploy
   tags:
-    - master.unturf.com  # Change to match your runner’s tags
+    - master.unturf.com
   only:
     - main  # Deploy only from the 'main' branch
   script:
-    # Step 1: Define variables for systemd service
+    # Step 1: Define variables with substitutions
+    - |
+      SERVICE_NAME="uwsgi-${BACKEND_APP_NAME//./-}.service"  # Replace dots with dashes
+      UWSGI_SOCKET="${BACKEND_APP_DIR}/uwsgi.sock"           # Place socket in app directory
+
+    # Step 2: Synchronize the code to the backend application directory
+    - |
+      if [ ! -d "${BACKEND_APP_DIR}" ]; then
+        mkdir -p "${BACKEND_APP_DIR}"
+      fi
+      rsync -a --exclude=data/ --exclude=venv/ --delete "${CI_PROJECT_DIR}/" "${BACKEND_APP_DIR}/"
+
+    # Step 3: Set up Python virtual environment and install dependencies
+    - |
+      if [ ! -d "${BACKEND_VENV_DIR}" ]; then
+        python3 -m venv "${BACKEND_VENV_DIR}"
+      fi
+      source "${BACKEND_VENV_DIR}/bin/activate"
+      pip install --upgrade pip
+      pip install -r "${BACKEND_APP_DIR}/requirements.txt"
+
+    # Step 4: Install uWSGI in virtual environment
+    - |
+      source "${BACKEND_VENV_DIR}/bin/activate"
+      pip install uwsgi
+
+    # Step 5: Create or update the systemd user service file for uWSGI
+    - |
+      mkdir -p ~/.config/systemd/user/
+
+      # Set the MODULE for slop_with_models.py
+      MODULE="slop_with_models:app"
+
+      cat > ~/.config/systemd/user/${SERVICE_NAME} <<EOF
+      [Unit]
+      Description=uWSGI instance to serve ${BACKEND_APP_NAME}
+      After=network.target
+
+      [Service]
+      WorkingDirectory=${BACKEND_APP_DIR}
+      Environment="PATH=${BACKEND_VENV_DIR}/bin"
+      Environment="MODEL_ENDPOINT_1=https://hermes.ai.unturf.com/v1"
+      Environment="MODEL_ENDPOINT_2=https://node2.naptha.ai/inference"
+      Environment="MODEL_ENDPOINT_3=https://node3.naptha.ai/inference"
+      Environment="MODEL_API_KEY_1=${MODEL_API_KEY_1:-not-needed}"
+      Environment="MODEL_API_KEY_2=${MODEL_API_KEY_2:-not-needed}"
+      Environment="MODEL_API_KEY_3=${MODEL_API_KEY_3:-not-needed}"
+      ExecStart=${BACKEND_VENV_DIR}/bin/uwsgi \\
+        --master \\
+        --enable-threads \\
+        --processes 4 \\
+        --http-socket ${UWSGI_SOCKET} \\
+        --chmod-socket=660 \\
+        --chown-socket=gitlab-runner:caddy \\
+        --vacuum \\
+        --die-on-term \\
+        --chdir ${BACKEND_APP_DIR} \\
+        --logto2 ${BACKEND_APP_DIR}/uwsgi.log \\
+        --module ${MODULE}
+
+      Restart=always
+      KillSignal=SIGQUIT
+      Type=notify
+      NotifyAccess=all
+
+      [Install]
+      WantedBy=default.target
+      EOF
+
+    # Step 6: Reload systemd user daemon and restart the uWSGI service
+    - systemctl --user daemon-reload
+    - systemctl --user restart ${SERVICE_NAME}
+    - systemctl --user enable ${SERVICE_NAME}
+
+    # Step 7: Handle Caddy configurations for backend
+    - |
+      mkdir -p "${CADDY_SNIPPETS_DIR}"
+      cat > "${CADDY_SNIPPETS_DIR}/${BACKEND_APP_NAME}.caddy" <<EOF
+      ${BACKEND_APP_NAME} {
+          encode gzip
+          tls {
+              on_demand
+          }
+          handle {
+              reverse_proxy unix//${UWSGI_SOCKET}
+          }
+      }
+      EOF
+
+    # Deployment completed
+    - echo "Backend deployment completed successfully."
+
+deploy-streamlit:
+  stage: deploy-streamlit
+  tags:
+    - master.unturf.com
+  only:
+    - main  # Deploy only from the 'main' branch
+  script:
+    # Step 1: Define variables
     - |
       SERVICE_NAME="streamlit-${STREAMLIT_APP_NAME//./-}.service"  # Replace dots with dashes
-      STREAMLIT_PORT="8501"  # Default Streamlit port
+      STREAMLIT_PORT="8501"  # Default Streamlit port, can be changed
 
     # Step 2: Synchronize the code to the Streamlit application directory
     - |
@@ -40,8 +137,7 @@ deploy:
       fi
       source "${STREAMLIT_VENV_DIR}/bin/activate"
       pip install --upgrade pip
-      # If you have a requirements.txt, uncomment the line below:
+      pip install -r "${STREAMLIT_APP_DIR}/requirements.txt"
-      # pip install -r "${STREAMLIT_APP_DIR}/requirements.txt"
       pip install streamlit
 
     # Step 4: Create or update the systemd user service file for Streamlit
@@ -50,15 +146,13 @@ deploy:
 
       cat > ~/.config/systemd/user/${SERVICE_NAME} <<EOF
       [Unit]
-      Description=Streamlit service for ${STREAMLIT_APP_NAME}
+      Description=Streamlit instance to serve ${STREAMLIT_APP_NAME}
       After=network.target
 
       [Service]
       WorkingDirectory=${STREAMLIT_APP_DIR}
       Environment="PATH=${STREAMLIT_VENV_DIR}/bin"
-      ExecStart=${STREAMLIT_VENV_DIR}/bin/streamlit run ${STREAMLIT_SCRIPT_FILENAME} \\
+      ExecStart=${STREAMLIT_VENV_DIR}/bin/streamlit run streamlit_slop_with_models.py --server.port ${STREAMLIT_PORT} --server.address 127.0.0.1
-                --server.port ${STREAMLIT_PORT} \\
-                --server.address 127.0.0.1
       Restart=always
       Type=simple
 
@@ -86,13 +180,15 @@ deploy:
       }
       EOF
 
-      # Combine all snippet files into one main Caddyfile
+      # Concatenate all snippets into the main Caddyfile
       echo '# Main Caddyfile - concatenated snippets' > "${CADDYFILE_DIR}/Caddyfile"
       cat "${CADDY_SNIPPETS_DIR}"/*.caddy >> "${CADDYFILE_DIR}/Caddyfile"
 
-      # Validate the final Caddyfile
+      # Validate the Caddyfile
       caddy validate --config "${CADDYFILE_DIR}/Caddyfile"
 
+    # Step 7: Reload Caddy service
     - sudo /bin/systemctl restart caddy.service
 
-    - echo "Streamlit deployment for ${STREAMLIT_APP_NAME} completed successfully."
+    # Deployment completed
+    - echo "Streamlit deployment completed successfully."