diff --git a/journal.rst b/journal.rst index 3526fbb..abc17fa 100644 --- a/journal.rst +++ b/journal.rst @@ -285,3 +285,46 @@ I am honestly having a hard time working on this "physical product" feature & i' I have a desire to table this project, temporarily at least. + +Tue Sep 13 03:22:14 PM EDT 2022 +======================================= + +This is a script to recompute the file_bytes & total_file_bytes of every Product object of the database. + +It will query the s3 backend and fix our meta data regarding content length for our capacity usage. + + +``` + +import botocore.exceptions + +# begin the database transaction. +request.tm.begin() + +all_products = models.get_all_products(request.dbsession) + +for product in all_products: + for file_key in product.file_keys: + + try: + response = request.secure_uploads_client.head_object( + Bucket=request.app["bucket.secure_uploads"], + Key="{}/{}".format(product.s3_path, file_key) + ) + except botocore.exceptions.ClientError: + # skip the rest of this iteration. + continue + + product_content_length = response["ContentLength"] + + tmp_file_bytes = product.file_bytes + tmp_file_bytes[file_key] = product_content_length + product.file_bytes = tmp_file_bytes + + request.dbsession.add(product) + request.dbsession.flush() + +# commit/close the database transaction to really make changes. +request.tm.commit() + +``` diff --git a/make_post_sell/models/product.py b/make_post_sell/models/product.py index f8313ba..a895107 100644 --- a/make_post_sell/models/product.py +++ b/make_post_sell/models/product.py @@ -73,6 +73,10 @@ class Product(RBase, Base): json_file_metadata = Column(UnicodeText, default=DEFAULT_FILE_METADATA) + json_file_bytes = Column(UnicodeText, default="{}") + + total_file_bytes = Column(BigInteger, nullable=False, default=0) + # a denormalized json field to relate products to bundles, and vice versa. json_bundle_metadata = Column( UnicodeText, default=DEFAULT_BUNDLE_METADATA, nullable=False @@ -265,6 +269,15 @@ class Product(RBase, Base): return self._file_metadata return {} + @property + def file_bytes(self): + return json.loads(self.json_file_bytes) + + @file_bytes.setter + def file_bytes(self, new_dict): + self.json_file_bytes = json.dumps(new_dict) + self.total_file_bytes = sum(self.file_bytes.values()) + @property def extensions(self): return self.file_metadata.get("extensions", {}) @@ -274,7 +287,6 @@ class Product(RBase, Base): return self.file_metadata.get("originals", {}) def set_file_metadata(self, file_key, extension, original_filename): - self.updated_timestamp = now_timestamp() tmp = self.file_metadata tmp["extensions"][file_key] = extension tmp["originals"][file_key] = original_filename diff --git a/make_post_sell/scripts/alembic/versions/7217f39a20a8_product_file_bytes.py b/make_post_sell/scripts/alembic/versions/7217f39a20a8_product_file_bytes.py new file mode 100644 index 0000000..bc14339 --- /dev/null +++ b/make_post_sell/scripts/alembic/versions/7217f39a20a8_product_file_bytes.py @@ -0,0 +1,28 @@ +"""product file bytes + +Revision ID: 7217f39a20a8 +Revises: d38d649cb682 +Create Date: 2022-09-13 14:57:57.100700 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '7217f39a20a8' +down_revision = 'd38d649cb682' +branch_labels = None +depends_on = None + +from make_post_sell.models.meta import UUIDType + + +def upgrade(): + op.add_column('mps_product', sa.Column('json_file_bytes', sa.UnicodeText(), nullable=True, server_default="{}")) + op.add_column('mps_product', sa.Column('total_file_bytes', sa.BigInteger(), nullable=False, server_default="0")) + + +def downgrade(): + op.drop_column('mps_product', 'total_file_bytes') + op.drop_column('mps_product', 'json_file_bytes') diff --git a/make_post_sell/views/product.py b/make_post_sell/views/product.py index 348e1e5..db12a62 100644 --- a/make_post_sell/views/product.py +++ b/make_post_sell/views/product.py @@ -272,7 +272,19 @@ def product_edit(request): Bucket=s3_webhook_bucket, Key=s3_webhook_key, ) - + + # get file size & store in our database. + response = request.secure_uploads_client.head_object( + Bucket=request.app["bucket.secure_uploads"], + Key="{}/{}".format(product.s3_path, file_key) + ) + + product_content_length = response["ContentLength"] + + tmp_file_bytes = product.file_bytes + tmp_file_bytes[file_key] = product_content_length + product.file_bytes = tmp_file_bytes + product.stamp_updated_timestamp() request.dbsession.add(product) request.dbsession.flush()