From aab74dff669af02b1053a4b21e35ab5847d09b87 Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Sun, 8 Dec 2024 15:02:15 -0500 Subject: [PATCH] MegaParce is now MegaFarce new file: Dockerfile new file: LICENSE new file: README.md new file: app.py new file: docker-compose.yml new file: pyproject.toml new file: requirements.lock --- Dockerfile | 36 ++ LICENSE | 201 +++++++++++ README.md | 46 +++ app.py | 811 +++++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 27 ++ pyproject.toml | 90 +++++ requirements.lock | 576 ++++++++++++++++++++++++++++++++ 7 files changed, 1787 insertions(+) create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.md create mode 100644 app.py create mode 100644 docker-compose.yml create mode 100644 pyproject.toml create mode 100644 requirements.lock diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5e4dc0b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# Dockerfile + +FROM python:3.11.10-slim-bullseye + +WORKDIR /app + +# Install runtime dependencies +RUN apt-get update && apt-get upgrade -y && apt-get install -y \ + libgeos-dev \ + libcurl4-openssl-dev \ + libssl-dev \ + binutils \ + curl \ + git \ + autoconf \ + automake \ + build-essential \ + libtool \ + python-dev \ + wget \ + gcc \ + libmagic-dev \ + poppler-utils \ + tesseract-ocr \ + libreoffice \ + libpq-dev \ + pandoc && \ + rm -rf /var/lib/apt/lists/* && apt-get clean + +COPY . . + +RUN pip install --no-cache-dir -r requirements.lock + +EXPOSE 8001 + +CMD ["python", "app.py"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2bbac88 --- /dev/null +++ b/README.md @@ -0,0 +1,46 @@ +# MegaFarse - Your Parser for every type of documents + +this is a MegaFarse fork by the community. + +MegaFarse is a powerful and versatile parser that can handle various types of documents with ease. Whether you're dealing with text, PDFs, Powerpoint presentations, Word documents MegaFarse has got you covered. Focus on having no information loss during parsing. + +## Key Features 🎯 + +- **Versatile Parser**: MegaFarse is a powerful and versatile parser that can handle various types of documents with ease. +- **No Information Loss**: Focus on having no information loss during parsing. +- **Fast and Efficient**: Designed with speed and efficiency at its core. +- **Wide File Compatibility**: Supports Text, PDF, Powerpoint presentations, Excel, CSV, Word documents. +- **Open Source**: Freedom is beautiful, and so is MegaFarse. Open source and free to use. + +## Support + +- Files: ✅ PDF ✅ Powerpoint ✅ Word +- Content: ✅ Tables ✅ TOC ✅ Headers ✅ Footers ✅ Images + +## Installation + +Docker only for now... + +``` +docker build -t megaparse:latest . +docker compose -f docker-compose.yml up --build +``` + +at the root of the project and you are good to go. + +See localhost:8001/docs for more info on the different endpoints ! + +## BenchMark + + +| Parser | similarity_ratio | +| ----------------------------- | ---------------- | +| megaparse_vision | 0.87 | +| unstructured_with_check_table | 0.77 | +| unstructured | 0.59 | +| llama_parser | 0.33 | + + +_Higher the better_ + +Note: Want to evaluate and compare your MegaFarse module with ours ? Please add your config in ```evaluations/script.py``` and then run ```python evaluations/script.py```. If it is better, do a PR, I mean, let's go higher together! diff --git a/app.py b/app.py new file mode 100644 index 0000000..39203c4 --- /dev/null +++ b/app.py @@ -0,0 +1,811 @@ +# app.py + +import asyncio +import base64 +import enum +import io +import os +import re +import tempfile +import time +from enum import Enum +from pathlib import Path +from typing import Any, Dict, IO, List, Optional, Union + +from abc import ABC, abstractmethod + +import httpx +import nats +import psutil +import uvicorn +from dotenv import load_dotenv +from fastapi import ( + Depends, + FastAPI, + File, + Form, + HTTPException, + UploadFile, +) +from langchain_anthropic import ChatAnthropic +from langchain_community.document_loaders import PlaywrightURLLoader +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.prompts import ChatPromptTemplate +from langchain_openai import ChatOpenAI +from llama_index.core.schema import Document as LlamaDocument +from llama_parse import LlamaParse as _LlamaParse +from llama_parse.utils import Language, ResultType +#from megaparse_sdk.schema.languages import Language as MP_Language +from pydantic import BaseModel, Field, ValidationError, validator + + + +# Load environment variables +load_dotenv() + + +# ---------------------- Langauges -------------------------- + +from enum import Enum + + +class Language(str, Enum): + BAZA = "abq" + ADYGHE = "ady" + AFRIKAANS = "af" + ANGIKA = "ang" + ARABIC = "ar" + ASSAMESE = "as" + AVAR = "ava" + AZERBAIJANI = "az" + BELARUSIAN = "be" + BULGARIAN = "bg" + BIHARI = "bh" + BHOJPURI = "bho" + BENGALI = "bn" + BOSNIAN = "bs" + SIMPLIFIED_CHINESE = "ch_sim" + TRADITIONAL_CHINESE = "ch_tra" + CHECHEN = "che" + CZECH = "cs" + WELSH = "cy" + DANISH = "da" + DARGWA = "dar" + GERMAN = "de" + ENGLISH = "en" + SPANISH = "es" + ESTONIAN = "et" + PERSIAN_FARSI = "fa" + FRENCH = "fr" + IRISH = "ga" + GOAN_KONKANI = "gom" + HINDI = "hi" + CROATIAN = "hr" + HUNGARIAN = "hu" + INDONESIAN = "id" + INGUSH = "inh" + ICELANDIC = "is" + ITALIAN = "it" + JAPANESE = "ja" + KABARDIAN = "kbd" + KANNADA = "kn" + KOREAN = "ko" + KURDISH = "ku" + LATIN = "la" + LAK = "lbe" + LEZGHIAN = "lez" + LITHUANIAN = "lt" + LATVIAN = "lv" + MAGAHI = "mah" + MAITHILI = "mai" + MAORI = "mi" + MONGOLIAN = "mn" + MARATHI = "mr" + MALAY = "ms" + MALTESE = "mt" + NEPALI = "ne" + NEWARI = "new" + DUTCH = "nl" + NORWEGIAN = "no" + OCCITAN = "oc" + PALI = "pi" + POLISH = "pl" + PORTUGUESE = "pt" + ROMANIAN = "ro" + RUSSIAN = "ru" + SERBIAN_CYRILLIC = "rs_cyrillic" + SERBIAN_LATIN = "rs_latin" + NAGPURI = "sck" + SLOVAK = "sk" + SLOVENIAN = "sl" + ALBANIAN = "sq" + SWEDISH = "sv" + SWAHILI = "sw" + TAMIL = "ta" + TABASSARAN = "tab" + TELUGU = "te" + THAI = "th" + TAJIK = "tjk" + TAGALOG = "tl" + TURKISH = "tr" + UYGHUR = "ug" + UKRAINIAN = "uk" + URDU = "ur" + UZBEK = "uz" + VIETNAMESE = "vi" + +MP_Language = Language + +# --------------------- Exception Classes --------------------- + +class HTTPModelNotSupported(HTTPException): + def __init__( + self, + detail: str = "The requested model is not supported yet.", + headers: Dict[str, Any] | None = None, + ): + super().__init__(status_code=501, detail=detail, headers=headers) + + +class HTTPFileNotFound(HTTPException): + def __init__( + self, + message="The UploadFile.filename does not exist and is needed for this operation", + ): + super().__init__(status_code=404, detail=message) + + +class HTTPDownloadError(HTTPException): + def __init__(self, file_name: str, message: str = "Failed to download the file"): + message = f"{file_name} : {message}" + super().__init__(status_code=400, detail=message) + + +class HTTPParsingException(HTTPException): + def __init__(self, file_name: str, message: str = "Failed to parse the file"): + message = f"{file_name} : {message}" + super().__init__(status_code=500, detail=message) + + +class ParsingException(Exception): + """Exception raised for errors in the parsing process.""" + + def __init__(self, message: str = "An error occurred during parsing"): + self.message = message + super().__init__(self.message) + +# --------------------- Pydantic Models --------------------- + +class MarkDownType(str, Enum): + """Markdown type enumeration.""" + + TITLE = "Title" + SUBTITLE = "Subtitle" + HEADER = "Header" + FOOTER = "Footer" + NARRATIVE_TEXT = "NarrativeText" + LIST_ITEM = "ListItem" + TABLE = "Table" + PAGE_BREAK = "PageBreak" + IMAGE = "Image" + FORMULA = "Formula" + FIGURE_CAPTION = "FigureCaption" + ADDRESS = "Address" + EMAIL_ADDRESS = "EmailAddress" + CODE_SNIPPET = "CodeSnippet" + PAGE_NUMBER = "PageNumber" + DEFAULT = "Default" + UNDEFINED = "Undefined" + + +class ParserType(str, Enum): + """Parser type enumeration.""" + + UNSTRUCTURED = "unstructured" + LLAMA_PARSER = "llama_parser" + MEGAPARSE_VISION = "megaparse_vision" + + +class StrategyEnum(str, Enum): + """Method to use for the conversion""" + + FAST = "fast" + AUTO = "auto" + HI_RES = "hi_res" + + +class SupportedModel(str, Enum): + """Supported models enumeration.""" + + GPT_4O = "gpt-4o" + GPT_4O_MINI = "gpt-o1-mini" + CLAUDE_3_5_SONNET = "claude-3-5-sonnet" + CLAUDE_3_OPUS = "claude-3-opus" + + def __str__(self): + return self.value + + @classmethod + def is_supported(cls, model_name: str) -> bool: + """Check if the model is supported.""" + return model_name in cls.__members__.values() + + +class APIOutputType(str, Enum): + PARSE_OK = "parse_file_ok" + PARSE_ERR = "parse_file_err" + + +class APIOutput(BaseModel): + message: str + result: str + + +class UploadFileConfig(BaseModel): + method: ParserType = ParserType.UNSTRUCTURED + strategy: StrategyEnum = StrategyEnum.AUTO + check_table: bool = False + language: Language = Language.ENGLISH + parsing_instruction: Optional[str] = None + model_name: SupportedModel = SupportedModel.GPT_4O + + @validator("model_name") + def validate_model(cls, v): + if not SupportedModel.is_supported(v.value): + raise ValueError("Unsupported model selected.") + return v + + +# --------------------- Parser Classes --------------------- + +class BaseParser(ABC): + """Mother Class for all the parsers [Unstructured, LlamaParse, MegaParseVision]""" + + @abstractmethod + async def convert( + self, + file_path: str | Path | None = None, + file: IO[bytes] | None = None, + **kwargs, + ) -> str: + """ + Convert the given file to a specific format. + + Args: + file_path (str | Path): The path to the file to be converted. + **kwargs: Additional keyword arguments for the conversion process. + + Returns: + str: The result of the conversion process. + + Raises: + NotImplementedError: If the method is not implemented by a subclass. + """ + raise NotImplementedError("Subclasses should implement this method") + + +class UnstructuredParser(BaseParser): + def __init__( + self, strategy=StrategyEnum.AUTO, model: Optional[BaseChatModel] = None, **kwargs + ): + self.strategy = strategy + self.model = model + + # Function to convert element category to markdown format + def convert_to_markdown(self, elements: List[Dict[str, Any]]) -> str: + markdown_content = "" + + for el in elements: + markdown_content += self.get_markdown_line(el) + + return markdown_content + + def get_markdown_line(self, el: Dict[str, Any]) -> str: + element_type = el["type"] + text = el["text"] + metadata = el["metadata"] + parent_id = metadata.get("parent_id", None) + category_depth = metadata.get("category_depth", 0) + + # Markdown line defaults to empty + markdown_line = "" + + # Element type-specific markdown content + markdown_types = { + "Title": f"## {text}\n\n" if parent_id else f"# {text}\n\n", + "Subtitle": f"## {text}\n\n", + "Header": f"{'#' * (category_depth + 1)} {text}\n\n", + "Footer": f"#### {text}\n\n", + "NarrativeText": f"{text}\n\n", + "ListItem": f"- {text}\n", + "Table": f"{text}\n\n", + "PageBreak": "---\n\n", + "Image": f"![Image]({el['metadata'].get('image_path', '')})\n\n", + "Formula": f"$$ {text} $$\n\n", + "FigureCaption": f"**Figure:** {text}\n\n", + "Address": f"**Address:** {text}\n\n", + "EmailAddress": f"**Email:** {text}\n\n", + "CodeSnippet": f"```{el['metadata'].get('language', '')}\n{text}\n```\n\n", + "PageNumber": "", # Page number is not included in markdown + } + + markdown_line = markdown_types.get(element_type, f"{text}\n\n") + + if element_type == "Table" and self.model: + # FIXME: @Chloé - Add a modular table enhancement here - LVM + prompt = ChatPromptTemplate.from_messages( + [ + ( + "human", + """You are an expert in markdown tables, match this text and this html table to fill a md table. You answer with just the table in pure markdown, nothing else. + + {text} + + + {html} + + + {previous_table} + """, + ), + ] + ) + chain = prompt | self.model + result = chain.invoke( + { + "text": el["text"], + "html": metadata["text_as_html"], + "previous_table": "", + } + ) + content_str = ( + str(result.content) + if not isinstance(result.content, str) + else result.content + ) + cleaned_content = re.sub(r"^```.*$\n?", "", content_str, flags=re.MULTILINE) + markdown_line = f"[TABLE]\n{cleaned_content}\n[/TABLE]\n\n" + + return markdown_line + + async def convert( + self, + file_path: str | Path | None = None, + file: IO[bytes] | None = None, + **kwargs, + ) -> str: + # Partition the PDF + elements = partition( + filename=str(file_path) if file_path else None, + file=file, + strategy=self.strategy, + skip_infer_table_types=[], + ) + elements_dict = [el.to_dict() for el in elements] + markdown_content = self.convert_to_markdown(elements_dict) + return markdown_content + + +class LlamaParser(BaseParser): + def __init__( + self, + api_key: str, + verbose: bool = True, + language: Language = Language.ENGLISH, + parsing_instruction: Optional[str] = None, + **kwargs, + ): + self.api_key = api_key + self.verbose = verbose + self.language = language + if parsing_instruction: + self.parsing_instruction = parsing_instruction + else: + self.parsing_instruction = """Do not take into account the page breaks (no --- between pages), + do not repeat the header and the footer so the tables are merged if needed. Keep the same format for similar tables.""" + + async def convert( + self, + file_path: str | Path | None = None, + file: IO[bytes] | None = None, + **kwargs, + ) -> str: + if not file_path: + raise ValueError("File_path should be provided to run LlamaParser") + + llama_parser = _LlamaParse( + api_key=self.api_key, + result_type=ResultType.MD, + gpt4o_mode=True, + verbose=self.verbose, + language=self.language, + parsing_instruction=self.parsing_instruction, + ) + + documents: List[LlamaDocument] = await llama_parser.aload_data(str(file_path)) + parsed_md = "" + for document in documents: + text_content = document.text + parsed_md = parsed_md + text_content + + return parsed_md + + +class MegaParseVision(BaseParser): + def __init__(self, model: BaseChatModel, **kwargs): + if hasattr(model, "model_name"): + if not SupportedModel.is_supported(model.model_name): + raise ValueError( + f"Invald model name, MegaParse vision only supports model that have vision capabilities. " + f"{model.model_name} is not supported." + ) + self.model = model + self.parsed_chunks: List[str] | None = None + + def process_file(self, file_path: str, image_format: str = "PNG") -> List[str]: + """ + Process a PDF file and convert its pages to base64 encoded images. + + :param file_path: Path to the PDF file + :param image_format: Format to save the images (default: PNG) + :return: List of base64 encoded images + """ + try: + images = convert_from_path(file_path) + images_base64 = [] + for image in images: + buffered = io.BytesIO() + image.save(buffered, format=image_format) + image_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8") + images_base64.append(image_base64) + return images_base64 + except Exception as e: + raise ValueError(f"Error processing PDF file: {str(e)}") + + def get_element(self, tag: Enum, chunk: str) -> List[str]: + pattern = rf"\[{tag.value}\]([\s\S]*?)\[/{tag.value}\]" + all_elmts = re.findall(pattern, chunk) + if not all_elmts: + print(f"No {tag.value} found in the chunk") + return [] + return [elmt.strip() for elmt in all_elmts] + + async def send_to_mlm(self, images_data: List[str]) -> str: + """ + Send images to the language model for processing. + + :param images_data: List of base64 encoded images + :return: Processed content as a string + """ + images_prompt = [ + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image_data}"}, + } + for image_data in images_data + ] + message = { + "content": [ + { + "type": "text", + "text": BASE_OCR_PROMPT, + }, + *images_prompt, + ] + } + response = await self.model.invoke([message]) + return str(response.content) + + def get_cleaned_content(self, parsed_file: str) -> str: + """ + Get cleaned parsed file without any tags defined in TagEnum. + + This method removes all tags from TagEnum from the parsed file, formats the content, + and handles the HEADER tag specially by keeping only the first occurrence. + + Args: + parsed_file (str): The parsed file content with tags. + + Returns: + str: The cleaned content without TagEnum tags. + """ + tag_pattern = "|".join(map(re.escape, TagEnum.__members__.values())) + tag_regex = rf"\[({tag_pattern})\](.*?)\[/\1\]" + # handle the HEADER tag specially + header_pattern = rf"\[{TagEnum.HEADER.value}\](.*?)\[/{TagEnum.HEADER.value}\]" + headers = re.findall(header_pattern, parsed_file, re.DOTALL) + if headers: + first_header = headers[0].strip() + # Remove all HEADER tags and their content + parsed_file = re.sub(header_pattern, "", parsed_file, flags=re.DOTALL) + # Add the first header back at the beginning + parsed_file = f"{first_header}\n{parsed_file}" + + # Remove all other tags + def remove_tag(match): + return match.group(2) + + cleaned_content = re.sub(tag_regex, remove_tag, parsed_file, flags=re.DOTALL) + + cleaned_content = re.sub(r"^```.*$\n?", "", cleaned_content, flags=re.MULTILINE) + cleaned_content = re.sub(r"\n\s*\n", "\n\n", cleaned_content) + cleaned_content = cleaned_content.replace("|\n\n|", "|\n|") + cleaned_content = cleaned_content.strip() + + return cleaned_content + + async def convert( + self, + file_path: str | Path | None = None, + file: IO[bytes] | None = None, + batch_size: int = 3, + **kwargs, + ) -> str: + """ + Parse a PDF file and process its content using the language model. + + :param file_path: Path to the PDF file + :param batch_size: Number of pages to process concurrently + :return: List of processed content strings + """ + if not file_path: + raise ValueError("File_path should be provided to run MegaParseVision") + + if isinstance(file_path, Path): + file_path = str(file_path) + pdf_base64 = self.process_file(file_path) + tasks = [ + self.send_to_mlm(pdf_base64[i : i + batch_size]) + for i in range(0, len(pdf_base64), batch_size) + ] + self.parsed_chunks = await asyncio.gather(*tasks) + responses = self.get_cleaned_content("\n".join(self.parsed_chunks)) + return responses + +# --------------------- MegaParse Class --------------------- + +class MegaParse: + def __init__( + self, + parser: BaseParser, + format_checker: Optional[Any] = None, + ) -> None: + self.parser = parser + self.format_checker = format_checker + self.last_parsed_document: str = "" + + async def aload( + self, + file_path: Path | str | None = None, + file: IO[bytes] | None = None, + file_extension: str | None = "", + ) -> str: + if not (file_path or file): + raise ValueError("Either file_path or file should be provided") + if file_path and file: + raise ValueError("Only one of file_path or file should be provided") + + if file_path: + if isinstance(file_path, str): + file_path = Path(file_path) + file_extension = file_path.suffix + elif file: + if not file_extension: + raise ValueError( + "file_extension should be provided when given file argument" + ) + file.seek(0) + + try: + FileExtension(file_extension) + except ValueError: + raise ValueError(f"Unsupported file extension: {file_extension}") + + if file_extension != ".pdf": + if self.format_checker: + raise ValueError( + f"Format Checker : Unsupported file extension: {file_extension}" + ) + if not isinstance(self.parser, UnstructuredParser): + raise ValueError( + f" Unsupported file extension : Parser {self.parser} do not support {file_extension}" + ) + + try: + parsed_document: str = await self.parser.convert( + file_path=file_path, file=file + ) + except Exception as e: + raise ParsingException(f"Error while parsing {file_path}: {e}") + + self.last_parsed_document = parsed_document + return parsed_document + + def load(self, file_path: Path | str) -> str: + if isinstance(file_path, str): + file_path = Path(file_path) + file_extension: str = file_path.suffix + + if file_extension != ".pdf": + if self.format_checker: + raise ValueError( + f"Format Checker : Unsupported file extension: {file_extension}" + ) + if not isinstance(self.parser, UnstructuredParser): + raise ValueError( + f"Parser {self.parser}: Unsupported file extension: {file_extension}" + ) + + try: + loop = asyncio.get_event_loop() + parsed_document: str = loop.run_until_complete( + self.parser.convert(file_path=file_path) + ) + except Exception as e: + raise ValueError(f"Error while parsing {file_path}: {e}") + + self.last_parsed_document = parsed_document + return parsed_document + + def save(self, file_path: Path | str) -> None: + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "w+") as f: + f.write(self.last_parsed_document) + +# --------------------- FastAPI App and Endpoints --------------------- + +app = FastAPI() + +playwright_loader = PlaywrightURLLoader(urls=[], remove_selectors=["header", "footer"]) + +def parser_builder_dep(): + return ParserBuilder() + +def get_playwright_loader(): + return playwright_loader + +@app.get("/healthz") +def healthz(): + return {"status": "ok"} + +def _check_free_memory() -> bool: + """Reject traffic when free memory is below minimum (default 2GB).""" + mem = psutil.virtual_memory() + memory_free_minimum = int(os.environ.get("MEMORY_FREE_MINIMUM_MB", 2048)) + + if mem.available <= memory_free_minimum * 1024 * 1024: + return False + return True + +@app.post( + "/v1/file", + response_model=APIOutput, +) +async def parse_file( + file: UploadFile = File(...), + method: ParserType = Form(ParserType.UNSTRUCTURED), + strategy: StrategyEnum = Form(StrategyEnum.AUTO), + check_table: bool = Form(False), + language: MP_Language = Form(MP_Language.ENGLISH), + parsing_instruction: Optional[str] = Form(None), + model_name: SupportedModel = Form(SupportedModel.GPT_4O), + parser_builder=Depends(parser_builder_dep), +) -> Dict[str, str]: + if not _check_free_memory(): + raise HTTPException( + status_code=503, detail="Service unavailable due to low memory" + ) + model = None + if model_name and check_table: + if model_name.value.startswith("gpt"): + model = ChatOpenAI(model=model_name.value, api_key=os.getenv("OPENAI_API_KEY")) # type: ignore + elif model_name.value.startswith("claude"): + model = ChatAnthropic( + model_name=model_name.value, + api_key=os.getenv("ANTHROPIC_API_KEY"), # type: ignore + timeout=60, + stop=None, + ) + else: + raise HTTPModelNotSupported() + + parser_config = { + "method": method, + "strategy": strategy, + "model": model if model and check_table else None, + "language": language, + "parsing_instruction": parsing_instruction, + } + try: + parser = ParserBuilder().build(parser_config) + megaparse = MegaParse(parser=parser) + if not file.filename: + raise HTTPFileNotFound("No filename provided") + _, extension = os.path.splitext(file.filename) + file_bytes = await file.read() + file_stream = io.BytesIO(file_bytes) + result = await megaparse.aload(file=file_stream, file_extension=extension) + return {"message": "File parsed successfully", "result": result} + except ParsingException as e: + print(e) + raise HTTPParsingException(file.filename) + except ValueError as e: + print(e) + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail=str(e)) + +@app.post( + "/v1/url", + response_model=APIOutput, +) +async def upload_url( + url: str, playwright_loader=Depends(get_playwright_loader) +) -> Dict[str, str]: + playwright_loader.urls = [url] + + if url.endswith(".pdf"): + # Download the file + async with httpx.AsyncClient() as client: + response = await client.get(url) + if response.status_code != 200: + raise HTTPDownloadError(url) + + with tempfile.NamedTemporaryFile(delete=False, suffix="pdf") as temp_file: + temp_file.write(response.content) + try: + megaparse = MegaParse( + parser=UnstructuredParser(strategy=StrategyEnum.AUTO) + ) + result = await megaparse.aload(temp_file.name) + return {"message": "File parsed successfully", "result": result} + except ParsingException: + raise HTTPParsingException(url) + else: + data = await playwright_loader.aload() + # Now turn the data into a string + extracted_content = "" + for page in data: + extracted_content += page.page_content + if not extracted_content: + raise HTTPDownloadError( + url, + message="Failed to extract content from the website. Valid URL example : https://www.quivr.com", + ) + return { + "message": "Website content parsed successfully", + "result": extracted_content, + } + +# --------------------- Parser Builder --------------------- + +class ParserBuilder: + parser_dict: Dict[str, BaseParser] = { + "unstructured": UnstructuredParser, + "llama_parser": LlamaParser, + "megaparse_vision": MegaParseVision, + } + + def build(self, config: Dict[str, Any]) -> BaseParser: + """ + Build a parser based on the given configuration. + + Args: + config (Dict): The configuration to be used for building the parser. + + Returns: + BaseParser: The built parser. + + Raises: + ValueError: If the configuration is invalid. + """ + parser_class = self.parser_dict.get(config["method"]) + if not parser_class: + raise ValueError(f"Unsupported parser method: {config['method']}") + return parser_class(**config) + +# --------------------- Runner --------------------- + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=8001) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..49bc89b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,27 @@ +# docker-compose.dev.yml + +version: "3.8" + +services: + megafarse: + build: + context: . + dockerfile: Dockerfile + cache_from: + - megaparse:latest + args: + - DEV_MODE=true + image: megafarse:latest + extra_hosts: + - "host.docker.internal:host-gateway" + container_name: megafarse + volumes: + - ./:/app/ + command: > + /bin/bash -c "python app.py" + restart: always + ports: + - 8001:8001 + environment: + - OPENAI_API_BASE=https://hermes.ai.unturf.com/v1 + - OPENAI_API_KEY=your_hermes_api_key_here diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..543b40c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,90 @@ +[project] +name = "megafarse-monorepo" +version = "0.0.1" +description = "MegaFarse monorepo" +authors = [ + { name = "Stan Girard", email = "stan@quivr.app" }, + { name = "Chloé Daems", email = "chloe@quivr.app" }, + { name = "Amine Dirhoussi", email = "amine@quivr.app" }, + { name = "Jacopo Chevallard", email = "jacopo@quivr.app" }, + { name = "Russell Ballestrini", email = "russell@ballestrini.net" }, +] +readme = "README.md" +requires-python = ">= 3.11" +dependencies = ["packaging>=22.0"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.rye] +python = ">= 3.11" +managed = true +universal = true +dev-dependencies = [ + "mypy>=1.11.1", + "pre-commit>=3.8.0", + "ipykernel>=6.29.5", + "ruff>=0.6.0", + "flake8>=7.1.1", + "flake8-black>=0.3.6", + "pytest-asyncio>=0.23.8", + "pytest>=8.3.3", + "pytest-xdist>=3.6.1", + "pytest-cov>=5.0.0", +] + +[tool.rye.workspace] +members = ["libs/*"] + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["src/megaparse"] + +[tool.ruff] +line-length = 88 +exclude = [".git", "__pycache__", ".mypy_cache", ".pytest_cache"] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "C", # flake8-comprehensions + "B", # flake8-bugbear +] +ignore = [ + "B904", + "B006", + "E501", # line too long, handled by black + "B008", # do not perform function calls in argument defaults + "C901", # too complex +] + +[tool.ruff.lint.isort] +order-by-type = true +relative-imports-order = "closest-to-furthest" +extra-standard-library = ["typing"] +section-order = [ + "future", + "standard-library", + "third-party", + "first-party", + "local-folder", +] +known-first-party = [] + + +[tool.pytest.ini_options] +addopts = "--tb=short -ra -v" +asyncio_default_fixture_loop_scope = "session" +filterwarnings = ["ignore::DeprecationWarning"] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "base: these tests require quivr-core with extra `base` to be installed", + "tika: these tests require a tika server to be running", + "unstructured: these tests require `unstructured` dependency", +] diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 0000000..2997464 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,576 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: true +# with-sources: false +# generate-hashes: false +# universal: true + +-e file:. + # via megaparse +aiohappyeyeballs==2.4.3 + # via aiohttp +aiohttp==3.11.5 + # via langchain + # via langchain-community + # via llama-index-core +aiosignal==1.3.1 + # via aiohttp +annotated-types==0.7.0 + # via pydantic +anthropic==0.39.0 + # via langchain-anthropic +antlr4-python3-runtime==4.9.3 + # via omegaconf +anyio==4.6.2.post1 + # via anthropic + # via httpx + # via openai + # via starlette +attrs==24.2.0 + # via aiohttp +backoff==2.2.1 + # via megaparse + # via unstructured +beautifulsoup4==4.12.3 + # via unstructured +cachetools==5.5.0 + # via google-auth +certifi==2024.8.30 + # via httpcore + # via httpx + # via requests +cffi==1.17.1 ; platform_python_implementation != 'PyPy' + # via cryptography +chardet==5.2.0 + # via unstructured +charset-normalizer==3.4.0 + # via pdfminer-six + # via requests +click==8.1.7 + # via llama-parse + # via nltk + # via python-oxmsg + # via uvicorn +colorama==0.4.6 ; sys_platform == 'win32' or platform_system == 'Windows' + # via click + # via loguru + # via tqdm +coloredlogs==15.0.1 + # via onnxruntime +contourpy==1.3.1 + # via matplotlib +cryptography==43.0.3 + # via pdfminer-six + # via unstructured-client +cycler==0.12.1 + # via matplotlib +dataclasses-json==0.6.7 + # via langchain-community + # via llama-index-core + # via unstructured +defusedxml==0.7.1 + # via langchain-anthropic +deprecated==1.2.15 + # via llama-index-core + # via pikepdf +dirtyjson==1.0.8 + # via llama-index-core +distro==1.9.0 + # via anthropic + # via openai +effdet==0.4.1 + # via unstructured +emoji==2.14.0 + # via unstructured +et-xmlfile==2.0.0 + # via openpyxl +eval-type-backport==0.2.0 + # via unstructured-client +fastapi==0.115.5 + # via megaparse +filelock==3.16.1 + # via huggingface-hub + # via torch + # via transformers + # via triton +filetype==1.2.0 + # via llama-index-core + # via unstructured +flatbuffers==24.3.25 + # via onnxruntime +fonttools==4.55.0 + # via matplotlib +frozenlist==1.5.0 + # via aiohttp + # via aiosignal +fsspec==2024.10.0 + # via huggingface-hub + # via llama-index-core + # via torch +google-api-core==2.23.0 + # via google-cloud-vision +google-auth==2.36.0 + # via google-api-core + # via google-cloud-vision +google-cloud-vision==3.8.1 + # via unstructured +googleapis-common-protos==1.66.0 + # via google-api-core + # via grpcio-status +greenlet==3.1.1 + # via playwright + # via sqlalchemy +grpcio==1.68.0 + # via google-api-core + # via grpcio-status +grpcio-status==1.68.0 + # via google-api-core +h11==0.14.0 + # via httpcore + # via uvicorn +httpcore==1.0.7 + # via httpx +httpx==0.27.2 + # via anthropic + # via langsmith + # via llama-index-core + # via megaparse-sdk + # via openai + # via unstructured-client +huggingface-hub==0.26.2 + # via timm + # via tokenizers + # via transformers + # via unstructured-inference +humanfriendly==10.0 + # via coloredlogs +idna==3.10 + # via anyio + # via httpx + # via requests + # via yarl +iopath==0.1.10 + # via layoutparser +jinja2==3.1.4 + # via torch +jiter==0.7.1 + # via anthropic + # via openai +joblib==1.4.2 + # via nltk +jsonpatch==1.33 + # via langchain-core +jsonpath-python==1.0.6 + # via unstructured-client +jsonpointer==3.0.0 + # via jsonpatch +kiwisolver==1.4.7 + # via matplotlib +langchain==0.2.17 + # via langchain-community + # via megaparse +langchain-anthropic==0.1.23 + # via megaparse +langchain-community==0.2.19 + # via megaparse +langchain-core==0.2.43 + # via langchain + # via langchain-anthropic + # via langchain-community + # via langchain-openai + # via langchain-text-splitters + # via megaparse +langchain-openai==0.1.25 + # via megaparse +langchain-text-splitters==0.2.4 + # via langchain +langdetect==1.0.9 + # via unstructured +langsmith==0.1.143 + # via langchain + # via langchain-community + # via langchain-core +layoutparser==0.3.4 + # via unstructured-inference +llama-index-core==0.12.0 + # via llama-parse +llama-parse==0.5.14 + # via megaparse +loguru==0.7.2 + # via megaparse-sdk +lxml==5.3.0 + # via pikepdf + # via python-docx + # via python-pptx + # via unstructured +markdown==3.7 + # via unstructured +markupsafe==3.0.2 + # via jinja2 +marshmallow==3.23.1 + # via dataclasses-json +matplotlib==3.9.2 + # via pycocotools + # via unstructured-inference +mpmath==1.3.0 + # via sympy +multidict==6.1.0 + # via aiohttp + # via yarl +mypy-extensions==1.0.0 + # via typing-inspect +nats-py==2.9.0 + # via megaparse + # via megaparse-sdk +nest-asyncio==1.6.0 + # via llama-index-core + # via unstructured-client +networkx==3.4.2 + # via llama-index-core + # via torch + # via unstructured +nltk==3.9.1 + # via llama-index-core + # via unstructured +numpy==1.26.4 + # via contourpy + # via langchain + # via langchain-community + # via layoutparser + # via llama-index-core + # via matplotlib + # via megaparse + # via onnx + # via onnxruntime + # via opencv-python + # via pandas + # via pycocotools + # via scipy + # via torchvision + # via transformers + # via unstructured +nvidia-cublas-cu12==12.4.5.8 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via nvidia-cudnn-cu12 + # via nvidia-cusolver-cu12 + # via torch +nvidia-cuda-cupti-cu12==12.4.127 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cuda-nvrtc-cu12==12.4.127 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cuda-runtime-cu12==12.4.127 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cudnn-cu12==9.1.0.70 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cufft-cu12==11.2.1.3 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-curand-cu12==10.3.5.147 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cusolver-cu12==11.6.1.9 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cusparse-cu12==12.3.1.170 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via nvidia-cusolver-cu12 + # via torch +nvidia-nccl-cu12==2.21.5 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-nvjitlink-cu12==12.4.127 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via nvidia-cufft-cu12 + # via nvidia-cusolver-cu12 + # via nvidia-cusparse-cu12 + # via torch +nvidia-nvtx-cu12==12.4.127 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +olefile==0.47 + # via python-oxmsg +omegaconf==2.3.0 + # via effdet +onnx==1.17.0 + # via unstructured + # via unstructured-inference +onnxruntime==1.20.0 + # via unstructured-inference +openai==1.54.5 + # via langchain-openai +opencv-python==4.10.0.84 + # via layoutparser + # via unstructured-inference +openpyxl==3.1.5 + # via unstructured +orjson==3.10.11 + # via langsmith +packaging==24.2 + # via huggingface-hub + # via langchain-core + # via marshmallow + # via matplotlib + # via megaparse-monorepo + # via onnxruntime + # via pikepdf + # via pytesseract + # via transformers + # via unstructured-pytesseract +pandas==2.2.3 + # via layoutparser + # via unstructured +pdf2image==1.17.0 + # via layoutparser + # via unstructured +pdfminer-six==20231228 + # via pdfplumber + # via unstructured +pdfplumber==0.11.4 + # via layoutparser + # via megaparse +pikepdf==9.4.2 + # via unstructured +pillow==11.0.0 + # via layoutparser + # via llama-index-core + # via matplotlib + # via pdf2image + # via pdfplumber + # via pikepdf + # via pillow-heif + # via pytesseract + # via python-pptx + # via torchvision + # via unstructured-pytesseract +pillow-heif==0.20.0 + # via unstructured +playwright==1.48.0 + # via megaparse +portalocker==3.0.0 + # via iopath +propcache==0.2.0 + # via aiohttp + # via yarl +proto-plus==1.25.0 + # via google-api-core + # via google-cloud-vision +protobuf==5.28.3 + # via google-api-core + # via google-cloud-vision + # via googleapis-common-protos + # via grpcio-status + # via onnx + # via onnxruntime + # via proto-plus +psutil==6.1.0 + # via megaparse + # via megaparse-sdk + # via unstructured +pyasn1==0.6.1 + # via pyasn1-modules + # via rsa +pyasn1-modules==0.4.1 + # via google-auth +pycocotools==2.0.8 + # via effdet +pycparser==2.22 ; platform_python_implementation != 'PyPy' + # via cffi +pycryptodome==3.21.0 + # via megaparse + # via megaparse-sdk +pydantic==2.9.2 + # via anthropic + # via fastapi + # via langchain + # via langchain-core + # via langsmith + # via llama-index-core + # via openai + # via pydantic-settings + # via unstructured-client +pydantic-core==2.23.4 + # via pydantic +pydantic-settings==2.6.1 + # via megaparse +pyee==12.0.0 + # via playwright +pypandoc==1.14 + # via unstructured +pyparsing==3.2.0 + # via matplotlib +pypdf==5.1.0 + # via megaparse + # via unstructured + # via unstructured-client +pypdfium2==4.30.0 + # via pdfplumber +pyreadline3==3.5.4 ; sys_platform == 'win32' + # via humanfriendly +pytesseract==0.3.13 + # via unstructured +python-dateutil==2.8.2 + # via matplotlib + # via pandas + # via unstructured-client +python-docx==1.1.2 + # via unstructured +python-dotenv==1.0.1 + # via megaparse + # via megaparse-sdk + # via pydantic-settings +python-iso639==2024.10.22 + # via unstructured +python-magic==0.4.27 + # via megaparse + # via unstructured +python-multipart==0.0.17 + # via unstructured-inference +python-oxmsg==0.0.1 + # via unstructured +python-pptx==0.6.23 + # via unstructured +pytz==2024.2 + # via pandas +pywin32==308 ; platform_system == 'Windows' + # via portalocker +pyyaml==6.0.2 + # via huggingface-hub + # via langchain + # via langchain-community + # via langchain-core + # via layoutparser + # via llama-index-core + # via omegaconf + # via timm + # via transformers +rapidfuzz==3.10.1 + # via unstructured + # via unstructured-inference +ratelimit==2.2.1 + # via megaparse +regex==2024.11.6 + # via nltk + # via tiktoken + # via transformers +requests==2.32.3 + # via google-api-core + # via huggingface-hub + # via langchain + # via langchain-community + # via langsmith + # via llama-index-core + # via requests-toolbelt + # via tiktoken + # via transformers + # via unstructured +requests-toolbelt==1.0.0 + # via langsmith + # via unstructured-client +rsa==4.9 + # via google-auth +safetensors==0.4.5 + # via timm + # via transformers +scipy==1.14.1 + # via layoutparser +setuptools==75.5.0 + # via torch +six==1.16.0 + # via langdetect + # via python-dateutil +sniffio==1.3.1 + # via anthropic + # via anyio + # via httpx + # via openai +soupsieve==2.6 + # via beautifulsoup4 +sqlalchemy==2.0.36 + # via langchain + # via langchain-community + # via llama-index-core +starlette==0.41.3 + # via fastapi +sympy==1.13.1 + # via onnxruntime + # via torch +tabulate==0.9.0 + # via unstructured +tenacity==8.5.0 + # via langchain + # via langchain-community + # via langchain-core + # via llama-index-core +tiktoken==0.8.0 + # via langchain-openai + # via llama-index-core +timm==1.0.11 + # via effdet + # via unstructured-inference +tokenizers==0.20.3 + # via transformers +torch==2.5.1 + # via effdet + # via timm + # via torchvision + # via unstructured-inference +torchvision==0.20.1 + # via effdet + # via timm +tqdm==4.67.0 + # via huggingface-hub + # via iopath + # via llama-index-core + # via nltk + # via openai + # via transformers + # via unstructured +transformers==4.46.3 + # via unstructured-inference +triton==3.1.0 ; python_full_version < '3.13' and platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +typing-extensions==4.12.2 + # via anthropic + # via fastapi + # via huggingface-hub + # via iopath + # via langchain-core + # via llama-index-core + # via openai + # via pydantic + # via pydantic-core + # via pyee + # via python-docx + # via python-oxmsg + # via sqlalchemy + # via torch + # via typing-inspect + # via unstructured +typing-inspect==0.9.0 + # via dataclasses-json + # via llama-index-core + # via unstructured-client +tzdata==2024.2 + # via pandas +unstructured==0.15.0 + # via megaparse +unstructured-client==0.27.0 + # via unstructured +unstructured-inference==0.7.36 + # via unstructured +unstructured-pytesseract==0.3.13 + # via unstructured +urllib3==2.2.3 + # via requests +uvicorn==0.32.0 + # via megaparse +uvloop==0.21.0 + # via megaparse +win32-setctime==1.1.0 ; sys_platform == 'win32' + # via loguru +wrapt==1.16.0 + # via deprecated + # via llama-index-core + # via unstructured +xlrd==2.0.1 + # via unstructured +xlsxwriter==3.2.0 + # via python-pptx +yarl==1.17.2 + # via aiohttp