engineering/make_post_sell
1
0
Fork 0
Code Issues 2 Pull requests Projects Releases 2 Packages Wiki Activity Actions

sunset miscutils!

Browse source 
new file:   make_post_sell/lib/hex_color.py
	modified:   make_post_sell/lib/mail.py
	modified:   make_post_sell/lib/render.py
	new file:   make_post_sell/lib/sanitize_html.py
	modified:   make_post_sell/models/cart.py
	modified:   make_post_sell/models/coupon.py
	modified:   make_post_sell/models/shop.py
	modified:   make_post_sell/models/user.py
	modified:   requirements.py3.txt
	modified:   requirements.txt
This commit is contained in:
Russell Ballestrini 2022-07-05 14:50:20 -04:00
parent 62a0b6ed93
commit ba6d2591b1
10 changed files with 411 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

43
make_post_sell/lib/hex_color.py Normal file
View file

@ -0,0 +1,43 @@
"""
Original Sourcecode pulled from:
https://thadeusb.com/weblog/2010/10/10/python_scale_hex_color/
"""
def clamp(val, minimum=0, maximum=255):
if val < minimum:
return minimum
if val > maximum:
return maximum
return val
def color_scale(hex_str, scale_factor):
"""
Scales a hex string by ``scale_factor``. Returns scaled hex string.
To darken the color, use a float value between 0 and 1.
To brighten the color, use a float value greater than 1.
>>> color_scale("#DF3C3C", .5)
#6F1E1E
>>> color_scale("#52D24F", 1.6)
#83FF7E
>>> color_scale("#4F75D2", 1)
#4F75D2
"""
hex_str = hex_str.strip("#")
if scale_factor < 0 or len(hex_str) != 6:
return hex_str
r, g, b = int(hex_str[:2], 16), int(hex_str[2:4], 16), int(hex_str[4:], 16)
r = int(clamp(r * scale_factor))
g = int(clamp(g * scale_factor))
b = int(clamp(b * scale_factor))
#return "#%02x%02x%02x" % (r, g, b)
return "#%02x%02x%02x" % (r, g, b)

114
make_post_sell/lib/mail.py
View file

@ -1,5 +1,3 @@
from miscutils.mail import send_pyramid_email
# quote email address in OTP so that a plus address
# is not decoded as a space during authentication.
try:
@ -22,6 +20,118 @@ from make_post_sell.lib.mail_messages import (
)
import dkim
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from datetime import datetime
# catch socket errors when postfix isn't running...
from socket import error as socket_error
def send_email(
to_email,
sender_email,
subject,
message_text,
message_html,
relay="localhost",
dkim_private_key_path="",
dkim_selector="",
dkim_signature_algorithm="ed25519-sha256",
):
# the `email` library assumes it is working with string objects.
# the `dkim` library assumes it is working with byte objects.
# this function performs the acrobatics to make them both happy.
if isinstance(message_text, bytes):
# needed for Python 3.
message_text = message_text.decode()
if isinstance(message_html, bytes):
# needed for Python 3.
message_html = message_html.decode()
sender_domain = sender_email.split("@")[-1]
msg = MIMEMultipart("alternative")
msg.attach(MIMEText(message_text, "plain"))
msg.attach(MIMEText(message_html, "html"))
msg["To"] = to_email
msg["From"] = sender_email
msg["Subject"] = subject
try:
# Python 3 libraries expect bytes.
msg_data = msg.as_bytes()
except:
# Python 2 libraries expect strings.
msg_data = msg.as_string()
if dkim_private_key_path and dkim_selector:
# the dkim library uses regex on byte strings so everything
# needs to be encoded from strings to bytes.
with open(dkim_private_key_path) as fh:
dkim_private_key = fh.read()
headers = [b"To", b"From", b"Subject"]
sig = dkim.sign(
message=msg_data,
selector=str(dkim_selector).encode(),
domain=sender_domain.encode(),
privkey=dkim_private_key.encode(),
include_headers=headers,
signature_algorithm=dkim_signature_algorithm.encode(),
)
# add the dkim signature to the email message headers.
# decode the signature back to string_type because later on
# the call to msg.as_string() performs it's own bytes encoding...
msg["DKIM-Signature"] = sig[len("DKIM-Signature: ") :].decode()
try:
# Python 3 libraries expect bytes.
msg_data = msg.as_bytes()
except:
# Python 2 libraries expect strings.
msg_data = msg.as_string()
# TODO: react if connecting to relay (localhost postfix) is a socket error.
s = smtplib.SMTP(relay)
s.sendmail(sender_email, [to_email], msg_data)
s.quit()
return msg
def send_pyramid_email(request, to_email, subject, message_text, message_html):
"""Thin wrapper around `send_email` to customise settings using request object."""
default_sender = "no-reply@{}".format(request.domain)
sender_email = request.app.get("email.sender", default_sender)
minute = datetime.now().strftime("%M")
subject = "{} | {} | {}".format(
subject, minute, request.app.get("email.subject_postfix", request.domain)
)
relay = request.app.get("email.relay", "localhost")
dkim_private_key_path = request.app.get("email.dkim_private_key_path", "")
dkim_selector = request.app.get("email.dkim_selector", "")
dkim_signature_algorithm = request.app.get("email.dkim_signature_algorithm", "ed25519-sha256")
send_email(
to_email,
sender_email,
subject,
message_text,
message_html,
relay,
dkim_private_key_path,
dkim_selector,
dkim_signature_algorithm,
)
def send_verification_digits_to_email(request, to_email, raw_digits):
"""
Send email with raw_digits a user may pass to verify & authenticate.

2
make_post_sell/lib/render.py
View file

@ -1,4 +1,4 @@
from miscutils.sanitize_html import (
from .sanitize_html import (
default_cleaner,
markdown_to_raw_html,
clean_raw_html,

227
make_post_sell/lib/sanitize_html.py Normal file
View file

@ -0,0 +1,227 @@
from markdown import markdown
from functools import partial
from collections import defaultdict
from bleach.sanitizer import Cleaner
from bleach.linkifier import LinkifyFilter
from bleach.callbacks import nofollow, target_blank
from bleach_allowlist import markdown_tags, markdown_attrs, all_styles
from bs4 import BeautifulSoup
import miniuri
import logging
log = logging.getLogger(__name__)
def default_tag_acl():
return defaultdict(list)
def default_cleaner(tag_acl=None):
"""
Returns a default Cleaner object.
We use BeautifulSoup to conditionally whitelist or blacklist
tags based on tag attr_name and attr_value pairs.
For example, this will whitelist Mathjax script tags:
tag_acl = {
"script": [
("type", "math/tex; mode=display", "allow"),
],
}
While this example will blacklist Mathjax script tags:
tag_acl = {
"script": [
("type", "math/tex; mode=display", "deny"),
],
}
"""
if tag_acl is None:
tag_acl = {}
maybe_safe_tags = ["pre", "table", "tr", "td"]
tags = maybe_safe_tags + list(tag_acl.keys()) + markdown_tags
attrs = markdown_attrs
attrs["img"].append("width")
attrs["span"] = ["class"]
# Allow both whitelist and blacklist tag_name/attr/attr_value
# to get past bleach.
# We will conditionally filter tags using BeautifulSoup.
for tag_name in tag_acl.keys():
# tag_name, for example "script".
for attr_name, attr_value, allow_or_deny in tag_acl[tag_name]:
if allow_or_deny == "allow":
# attr_name, for example "type".
# attr_value, for example "math/tex; mode=display".
if tag_name not in attrs:
# if tag_name not in attrs, create it as an empty list.
attrs[tag_name] = []
# append the attr_name to the map of approved
# attributes for the given tag_name.
attrs[tag_name].append(attr_name)
cleaner = Cleaner(tags=tags, attributes=attrs)
# doesn't do anything, but i used to be able to pass it via constructor.
# https://github.com/yourcelf/bleach-allowlist/blob/main/bleach_allowlist/bleach_allowlist.py
#cleaner.all_styles=all_styles
# disable link_protection by default.
cleaner.link_protection = False
# an None signifies a relative URI, which should always be whitelisted.
cleaner.whitelist_domains = [None]
# absolute domain, used for turning relative paths into absolute.
cleaner.absolute_domain = ""
# add conditional_whitelist_tags to cleaner object.
cleaner.tag_acl = tag_acl
return cleaner
def markdown_to_raw_html(data, extra_extensions=None):
"""Accepts a markdown string, returns raw unsanitized HTML"""
extensions = [
"markdown.extensions.codehilite",
"markdown.extensions.fenced_code",
]
if extra_extensions is not None:
extensions.extend(extra_extensions)
return markdown(data, extensions=extensions)
def conditional_tag_filter(soup, cleaner):
"""
Use BeautifulSoup `Soup` object to filter out non-whitelisted tags.
"""
# bleach does not have a way to conditionally accept tags whitelists.
# so we built our own way.
log.info(cleaner.tag_acl)
# for each tag_name in the tag_acl ...
for tag_name in cleaner.tag_acl.keys():
# find all the tags in the DOM that match the tag_name, for example "script".
for tag in soup.find_all(tag_name):
# tag is a BeautifulSoup tag object.
# by default, assume we will extract tag (remove it from the DOM).
extract_tag = True
for attr_name, attr_value, allow_or_deny in cleaner.tag_acl[tag_name]:
# tag attr_name:
#
# * "type"
# * "id"
# * "class"
# * ect
#
# tag attr_value:
#
# * "math/tex; mode=display"
# * "<uuid>"
# * ect
#
# allow_or_deny:
#
# * "allow"
# * "deny"
log.info("{} {} {}".format(attr_name, attr_value, allow_or_deny))
if allow_or_deny == "allow":
if tag.attrs.get(attr_name) == attr_value:
# this tag attr_name/attr_value is whitelisted.
# do not extract tag (remove it from the DOM)
extract_tag = False
break
if extract_tag:
log.info(
"Extracting {} tag with attr_name {} and attr_value {}".format(
tag_name, attr_name, attr_value
)
)
# remove this tag.
# this tag attr_name:attr_value is _not_ whitelisted.
tag.extract()
return soup
def protect_links(soup, cleaner):
for a_tag in soup.find_all("a"):
uri = miniuri.Uri(a_tag.attrs.get("href", ""))
if uri.hostname in cleaner.whitelist_domains:
# domain in whitelist or relative URI so remove rel="nofollow".
a_tag.attrs.pop("rel", None)
if uri.hostname is None and cleaner.absolute_domain:
# make relative path absolute!
# TODO: if we hold scheme in Namespace object we can use it
# when building absolute uris. for now assume https.
uri.scheme = "https"
uri.hostname = cleaner.absolute_domain
a_tag.attrs["href"] = str(uri)
elif cleaner.link_protection:
# domain not in whitelist, replace a_tag with "[link removed]".
a_tag.replace_with("[link removed]")
return soup
def clean_raw_html(raw_html, cleaner=None):
"""
Accepts raw HTML and a cleaner object
Returns sanitized HTML as bytes.
"""
if cleaner is None:
cleaner = default_cleaner()
if cleaner.link_protection == False:
cleaner.filters.append(
partial(
LinkifyFilter,
callbacks=[nofollow, target_blank],
skip_tags=["pre", "code"],
)
)
cleaned_html = cleaner.clean(raw_html)
soup = BeautifulSoup(cleaned_html, "html5lib")
# conditionally accept whitelisted tags, filter out the rest.
soup = conditional_tag_filter(soup, cleaner)
# protect links from abuse.
soup = protect_links(soup, cleaner)
return soup.decode(eventual_encoding="utf-8")

7
make_post_sell/models/cart.py
View file

@ -29,7 +29,6 @@ from .shop import get_shops_by_ids
from .cart_coupon import CartCoupon
from miscutils import timestamp_to_ago_string
from ..lib.currency import cents_to_dollars
@ -39,6 +38,12 @@ except:
from six import u as unicode
def timestamp_to_ago_string(timestamp):
"""Accepts a timestamp and returns a human readable string"""
from ago import human
return human(timestamp_to_datetime(timestamp), 2, abbreviate=True)
class Cart(RBase, Base):
"""This class tracks a User's shopping carts."""

13
make_post_sell/models/coupon.py
View file

@ -18,8 +18,6 @@ from .meta import (
from .cart_coupon import CartCoupon
from miscutils import datetime_to_timestamp, timestamp_to_datetime
from ..lib.currency import (
dollars_to_cents,
cents_to_dollars,
@ -28,6 +26,17 @@ from ..lib.currency import (
from datetime import datetime
def timestamp_to_datetime(timestamp):
"""Accepts a milliseconds timestamp integer and returns a datetime"""
return datetime.fromtimestamp(timestamp / 1000.0)
def datetime_to_timestamp(dt):
"""returns an integer timestamp in milliseconds"""
epoch_dt = datetime(1970, 1, 1)
return (dt - epoch_dt).total_seconds() * 1000
class Coupon(RBase, Base):
"""This class represents a coupon."""

4
make_post_sell/models/shop.py
View file

@ -14,9 +14,9 @@ from .stripe_user_shop import StripeUserShop
from slugify import slugify
from miscutils.hex_color import color_scale
from ..lib.hex_color import color_scale
from make_post_sell.lib.render import markdown_to_html
from ..lib.render import markdown_to_html
try:
unicode("")

11
make_post_sell/models/user.py
View file

@ -22,8 +22,6 @@ from .cart import Cart
from .cart import get_cart_by_id
from miscutils import generate_password
import logging
log = logging.getLogger(__name__)
@ -35,6 +33,15 @@ except:
from six import u as unicode
def generate_password(size=32):
"""Return a system generated password"""
from random import choice
letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
digits = "0123456789"
pool = letters + digits
return "".join([choice(pool) for i in range(size)])
class User(RBase, Base):
"""This class represents a user account."""

6
requirements.py3.txt
View file

@ -40,18 +40,12 @@ dkimpy[ed25519]
# python 3.
markdown
# my miscutils checked out via git as an editable package.
#-e git+git@github.com:russellballestrini/miscutils.git#egg=miscutils
git+https://github.com/russellballestrini/miscutils.git#egg=miscutils
# human readable timestamps.
ago
# miniuri: The Universal URI Parser
miniuri
# my miscutils package can sanitize and make HTML safe.
# that said, it requires the following:
# Bleach sanitizes MarkDown (removes HTML/Javascript) to prevent XSS.
bleach>=2.1.4
bleach-allowlist

6
requirements.txt
View file

@ -44,18 +44,12 @@ dkimpy[ed25519]
# python 2.
markdown==2.6.11
# my miscutils checked out via git as an editable package.
#-e git+git@github.com:russellballestrini/miscutils.git#egg=miscutils
git+https://github.com/russellballestrini/miscutils.git#egg=miscutils
# human readable timestamps.
ago
# miniuri is a tiny URI parser.
miniuri
# my miscutils package can sanitize and make HTML safe.
# that said, it requires the following:
# Bleach sanitizes MarkDown (removes HTML/Javascript) to prevent XSS.
bleach
#bleach-whitelist