diff --git a/make_post_sell/lib/hex_color.py b/make_post_sell/lib/hex_color.py new file mode 100644 index 0000000..b2865e4 --- /dev/null +++ b/make_post_sell/lib/hex_color.py @@ -0,0 +1,43 @@ +""" +Original Sourcecode pulled from: + +https://thadeusb.com/weblog/2010/10/10/python_scale_hex_color/ +""" + + +def clamp(val, minimum=0, maximum=255): + if val < minimum: + return minimum + if val > maximum: + return maximum + return val + + +def color_scale(hex_str, scale_factor): + """ + Scales a hex string by ``scale_factor``. Returns scaled hex string. + + To darken the color, use a float value between 0 and 1. + To brighten the color, use a float value greater than 1. + + >>> color_scale("#DF3C3C", .5) + #6F1E1E + >>> color_scale("#52D24F", 1.6) + #83FF7E + >>> color_scale("#4F75D2", 1) + #4F75D2 + """ + + hex_str = hex_str.strip("#") + + if scale_factor < 0 or len(hex_str) != 6: + return hex_str + + r, g, b = int(hex_str[:2], 16), int(hex_str[2:4], 16), int(hex_str[4:], 16) + + r = int(clamp(r * scale_factor)) + g = int(clamp(g * scale_factor)) + b = int(clamp(b * scale_factor)) + + #return "#%02x%02x%02x" % (r, g, b) + return "#%02x%02x%02x" % (r, g, b) diff --git a/make_post_sell/lib/mail.py b/make_post_sell/lib/mail.py index 8ba7238..87d8223 100644 --- a/make_post_sell/lib/mail.py +++ b/make_post_sell/lib/mail.py @@ -1,5 +1,3 @@ -from miscutils.mail import send_pyramid_email - # quote email address in OTP so that a plus address # is not decoded as a space during authentication. try: @@ -22,6 +20,118 @@ from make_post_sell.lib.mail_messages import ( ) +import dkim + +import smtplib + +from email.mime.multipart import MIMEMultipart + +from email.mime.text import MIMEText + +from datetime import datetime + +# catch socket errors when postfix isn't running... +from socket import error as socket_error + + +def send_email( + to_email, + sender_email, + subject, + message_text, + message_html, + relay="localhost", + dkim_private_key_path="", + dkim_selector="", + dkim_signature_algorithm="ed25519-sha256", +): + + # the `email` library assumes it is working with string objects. + # the `dkim` library assumes it is working with byte objects. + # this function performs the acrobatics to make them both happy. + + if isinstance(message_text, bytes): + # needed for Python 3. + message_text = message_text.decode() + + if isinstance(message_html, bytes): + # needed for Python 3. + message_html = message_html.decode() + + sender_domain = sender_email.split("@")[-1] + msg = MIMEMultipart("alternative") + msg.attach(MIMEText(message_text, "plain")) + msg.attach(MIMEText(message_html, "html")) + msg["To"] = to_email + msg["From"] = sender_email + msg["Subject"] = subject + + try: + # Python 3 libraries expect bytes. + msg_data = msg.as_bytes() + except: + # Python 2 libraries expect strings. + msg_data = msg.as_string() + + if dkim_private_key_path and dkim_selector: + # the dkim library uses regex on byte strings so everything + # needs to be encoded from strings to bytes. + with open(dkim_private_key_path) as fh: + dkim_private_key = fh.read() + headers = [b"To", b"From", b"Subject"] + sig = dkim.sign( + message=msg_data, + selector=str(dkim_selector).encode(), + domain=sender_domain.encode(), + privkey=dkim_private_key.encode(), + include_headers=headers, + signature_algorithm=dkim_signature_algorithm.encode(), + ) + # add the dkim signature to the email message headers. + # decode the signature back to string_type because later on + # the call to msg.as_string() performs it's own bytes encoding... + msg["DKIM-Signature"] = sig[len("DKIM-Signature: ") :].decode() + + try: + # Python 3 libraries expect bytes. + msg_data = msg.as_bytes() + except: + # Python 2 libraries expect strings. + msg_data = msg.as_string() + + # TODO: react if connecting to relay (localhost postfix) is a socket error. + s = smtplib.SMTP(relay) + s.sendmail(sender_email, [to_email], msg_data) + s.quit() + return msg + + +def send_pyramid_email(request, to_email, subject, message_text, message_html): + """Thin wrapper around `send_email` to customise settings using request object.""" + default_sender = "no-reply@{}".format(request.domain) + sender_email = request.app.get("email.sender", default_sender) + minute = datetime.now().strftime("%M") + subject = "{} | {} | {}".format( + subject, minute, request.app.get("email.subject_postfix", request.domain) + ) + relay = request.app.get("email.relay", "localhost") + dkim_private_key_path = request.app.get("email.dkim_private_key_path", "") + dkim_selector = request.app.get("email.dkim_selector", "") + dkim_signature_algorithm = request.app.get("email.dkim_signature_algorithm", "ed25519-sha256") + + send_email( + to_email, + sender_email, + subject, + message_text, + message_html, + relay, + dkim_private_key_path, + dkim_selector, + dkim_signature_algorithm, + ) + + def send_verification_digits_to_email(request, to_email, raw_digits): """ Send email with raw_digits a user may pass to verify & authenticate. diff --git a/make_post_sell/lib/render.py b/make_post_sell/lib/render.py index 16d0271..621a7cf 100644 --- a/make_post_sell/lib/render.py +++ b/make_post_sell/lib/render.py @@ -1,4 +1,4 @@ -from miscutils.sanitize_html import ( +from .sanitize_html import ( default_cleaner, markdown_to_raw_html, clean_raw_html, diff --git a/make_post_sell/lib/sanitize_html.py b/make_post_sell/lib/sanitize_html.py new file mode 100644 index 0000000..cc5dea3 --- /dev/null +++ b/make_post_sell/lib/sanitize_html.py @@ -0,0 +1,227 @@ +from markdown import markdown + +from functools import partial + +from collections import defaultdict + +from bleach.sanitizer import Cleaner +from bleach.linkifier import LinkifyFilter +from bleach.callbacks import nofollow, target_blank + +from bleach_allowlist import markdown_tags, markdown_attrs, all_styles + +from bs4 import BeautifulSoup + +import miniuri + +import logging + +log = logging.getLogger(__name__) + + +def default_tag_acl(): + return defaultdict(list) + + +def default_cleaner(tag_acl=None): + """ + Returns a default Cleaner object. + + We use BeautifulSoup to conditionally whitelist or blacklist + tags based on tag attr_name and attr_value pairs. + + For example, this will whitelist Mathjax script tags: + + tag_acl = { + "script": [ + ("type", "math/tex; mode=display", "allow"), + ], + } + + While this example will blacklist Mathjax script tags: + + tag_acl = { + "script": [ + ("type", "math/tex; mode=display", "deny"), + ], + } + + """ + if tag_acl is None: + tag_acl = {} + + maybe_safe_tags = ["pre", "table", "tr", "td"] + + tags = maybe_safe_tags + list(tag_acl.keys()) + markdown_tags + attrs = markdown_attrs + + attrs["img"].append("width") + attrs["span"] = ["class"] + + # Allow both whitelist and blacklist tag_name/attr/attr_value + # to get past bleach. + # We will conditionally filter tags using BeautifulSoup. + for tag_name in tag_acl.keys(): + + # tag_name, for example "script". + + for attr_name, attr_value, allow_or_deny in tag_acl[tag_name]: + + if allow_or_deny == "allow": + + # attr_name, for example "type". + # attr_value, for example "math/tex; mode=display". + + if tag_name not in attrs: + # if tag_name not in attrs, create it as an empty list. + attrs[tag_name] = [] + + # append the attr_name to the map of approved + # attributes for the given tag_name. + attrs[tag_name].append(attr_name) + + cleaner = Cleaner(tags=tags, attributes=attrs) + + # doesn't do anything, but i used to be able to pass it via constructor. + # https://github.com/yourcelf/bleach-allowlist/blob/main/bleach_allowlist/bleach_allowlist.py + #cleaner.all_styles=all_styles + + # disable link_protection by default. + cleaner.link_protection = False + # an None signifies a relative URI, which should always be whitelisted. + cleaner.whitelist_domains = [None] + # absolute domain, used for turning relative paths into absolute. + cleaner.absolute_domain = "" + # add conditional_whitelist_tags to cleaner object. + cleaner.tag_acl = tag_acl + + return cleaner + + +def markdown_to_raw_html(data, extra_extensions=None): + """Accepts a markdown string, returns raw unsanitized HTML""" + extensions = [ + "markdown.extensions.codehilite", + "markdown.extensions.fenced_code", + ] + if extra_extensions is not None: + extensions.extend(extra_extensions) + + return markdown(data, extensions=extensions) + + +def conditional_tag_filter(soup, cleaner): + """ + Use BeautifulSoup `Soup` object to filter out non-whitelisted tags. + """ + + # bleach does not have a way to conditionally accept tags whitelists. + # so we built our own way. + + log.info(cleaner.tag_acl) + + # for each tag_name in the tag_acl ... + for tag_name in cleaner.tag_acl.keys(): + + # find all the tags in the DOM that match the tag_name, for example "script". + for tag in soup.find_all(tag_name): + + # tag is a BeautifulSoup tag object. + + # by default, assume we will extract tag (remove it from the DOM). + extract_tag = True + + for attr_name, attr_value, allow_or_deny in cleaner.tag_acl[tag_name]: + + # tag attr_name: + # + # * "type" + # * "id" + # * "class" + # * ect + # + # tag attr_value: + # + # * "math/tex; mode=display" + # * "" + # * ect + # + # allow_or_deny: + # + # * "allow" + # * "deny" + + log.info("{} {} {}".format(attr_name, attr_value, allow_or_deny)) + + if allow_or_deny == "allow": + + if tag.attrs.get(attr_name) == attr_value: + # this tag attr_name/attr_value is whitelisted. + # do not extract tag (remove it from the DOM) + extract_tag = False + break + + if extract_tag: + log.info( + "Extracting {} tag with attr_name {} and attr_value {}".format( + tag_name, attr_name, attr_value + ) + ) + # remove this tag. + # this tag attr_name:attr_value is _not_ whitelisted. + tag.extract() + + return soup + + +def protect_links(soup, cleaner): + + for a_tag in soup.find_all("a"): + uri = miniuri.Uri(a_tag.attrs.get("href", "")) + + if uri.hostname in cleaner.whitelist_domains: + # domain in whitelist or relative URI so remove rel="nofollow". + a_tag.attrs.pop("rel", None) + if uri.hostname is None and cleaner.absolute_domain: + # make relative path absolute! + # TODO: if we hold scheme in Namespace object we can use it + # when building absolute uris. for now assume https. + uri.scheme = "https" + uri.hostname = cleaner.absolute_domain + a_tag.attrs["href"] = str(uri) + + elif cleaner.link_protection: + # domain not in whitelist, replace a_tag with "[link removed]". + a_tag.replace_with("[link removed]") + + return soup + + +def clean_raw_html(raw_html, cleaner=None): + """ + Accepts raw HTML and a cleaner object + Returns sanitized HTML as bytes. + """ + if cleaner is None: + cleaner = default_cleaner() + + if cleaner.link_protection == False: + cleaner.filters.append( + partial( + LinkifyFilter, + callbacks=[nofollow, target_blank], + skip_tags=["pre", "code"], + ) + ) + + cleaned_html = cleaner.clean(raw_html) + + soup = BeautifulSoup(cleaned_html, "html5lib") + + # conditionally accept whitelisted tags, filter out the rest. + soup = conditional_tag_filter(soup, cleaner) + + # protect links from abuse. + soup = protect_links(soup, cleaner) + + return soup.decode(eventual_encoding="utf-8") diff --git a/make_post_sell/models/cart.py b/make_post_sell/models/cart.py index 27644c7..ab208e7 100644 --- a/make_post_sell/models/cart.py +++ b/make_post_sell/models/cart.py @@ -29,7 +29,6 @@ from .shop import get_shops_by_ids from .cart_coupon import CartCoupon -from miscutils import timestamp_to_ago_string from ..lib.currency import cents_to_dollars @@ -39,6 +38,12 @@ except: from six import u as unicode +def timestamp_to_ago_string(timestamp): + """Accepts a timestamp and returns a human readable string""" + from ago import human + return human(timestamp_to_datetime(timestamp), 2, abbreviate=True) + + class Cart(RBase, Base): """This class tracks a User's shopping carts.""" diff --git a/make_post_sell/models/coupon.py b/make_post_sell/models/coupon.py index ed074d4..862e09c 100644 --- a/make_post_sell/models/coupon.py +++ b/make_post_sell/models/coupon.py @@ -18,8 +18,6 @@ from .meta import ( from .cart_coupon import CartCoupon -from miscutils import datetime_to_timestamp, timestamp_to_datetime - from ..lib.currency import ( dollars_to_cents, cents_to_dollars, @@ -28,6 +26,17 @@ from ..lib.currency import ( from datetime import datetime +def timestamp_to_datetime(timestamp): + """Accepts a milliseconds timestamp integer and returns a datetime""" + return datetime.fromtimestamp(timestamp / 1000.0) + + +def datetime_to_timestamp(dt): + """returns an integer timestamp in milliseconds""" + epoch_dt = datetime(1970, 1, 1) + return (dt - epoch_dt).total_seconds() * 1000 + + class Coupon(RBase, Base): """This class represents a coupon.""" diff --git a/make_post_sell/models/shop.py b/make_post_sell/models/shop.py index 7ea0ab7..80bbc0a 100644 --- a/make_post_sell/models/shop.py +++ b/make_post_sell/models/shop.py @@ -14,9 +14,9 @@ from .stripe_user_shop import StripeUserShop from slugify import slugify -from miscutils.hex_color import color_scale +from ..lib.hex_color import color_scale -from make_post_sell.lib.render import markdown_to_html +from ..lib.render import markdown_to_html try: unicode("") diff --git a/make_post_sell/models/user.py b/make_post_sell/models/user.py index 91ab879..e26c8f2 100644 --- a/make_post_sell/models/user.py +++ b/make_post_sell/models/user.py @@ -22,8 +22,6 @@ from .cart import Cart from .cart import get_cart_by_id -from miscutils import generate_password - import logging log = logging.getLogger(__name__) @@ -35,6 +33,15 @@ except: from six import u as unicode +def generate_password(size=32): + """Return a system generated password""" + from random import choice + letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + digits = "0123456789" + pool = letters + digits + return "".join([choice(pool) for i in range(size)]) + + class User(RBase, Base): """This class represents a user account.""" diff --git a/requirements.py3.txt b/requirements.py3.txt index 99b99b7..d1c5dc9 100644 --- a/requirements.py3.txt +++ b/requirements.py3.txt @@ -40,18 +40,12 @@ dkimpy[ed25519] # python 3. markdown -# my miscutils checked out via git as an editable package. -#-e git+git@github.com:russellballestrini/miscutils.git#egg=miscutils -git+https://github.com/russellballestrini/miscutils.git#egg=miscutils - # human readable timestamps. ago # miniuri: The Universal URI Parser miniuri -# my miscutils package can sanitize and make HTML safe. -# that said, it requires the following: # Bleach sanitizes MarkDown (removes HTML/Javascript) to prevent XSS. bleach>=2.1.4 bleach-allowlist diff --git a/requirements.txt b/requirements.txt index c7a5ec8..a291c9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,18 +44,12 @@ dkimpy[ed25519] # python 2. markdown==2.6.11 -# my miscutils checked out via git as an editable package. -#-e git+git@github.com:russellballestrini/miscutils.git#egg=miscutils -git+https://github.com/russellballestrini/miscutils.git#egg=miscutils - # human readable timestamps. ago # miniuri is a tiny URI parser. miniuri -# my miscutils package can sanitize and make HTML safe. -# that said, it requires the following: # Bleach sanitizes MarkDown (removes HTML/Javascript) to prevent XSS. bleach #bleach-whitelist