diff --git a/async_web_fetcher.py b/async_web_fetcher.py
index ae9c489..d82e397 100644
--- a/async_web_fetcher.py
+++ b/async_web_fetcher.py
@@ -44,10 +44,72 @@ IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp', '.
VIDEO_EXTENSIONS = {'.mp4', '.webm', '.mov', '.avi', '.mkv', '.m4v', '.ogv', '.flv', '.wmv'}
AUDIO_EXTENSIONS = {'.mp3', '.wav', '.ogg', '.m4a', '.flac', '.aac', '.wma'}
+# Code file extensions
+CODE_EXTENSIONS = {
+ '.py', '.js', '.ts', '.jsx', '.tsx', '.mjs', '.cjs', # Python, JavaScript, TypeScript
+ '.rs', '.go', '.rb', '.php', '.pl', '.pm', # Rust, Go, Ruby, PHP, Perl
+ '.java', '.kt', '.kts', '.scala', '.groovy', # JVM languages
+ '.c', '.h', '.cpp', '.hpp', '.cc', '.cxx', '.hxx', # C/C++
+ '.cs', '.fs', '.fsx', # .NET
+ '.swift', '.m', '.mm', # Apple
+ '.lua', '.r', '.R', '.jl', # Lua, R, Julia
+ '.sh', '.bash', '.zsh', '.fish', '.ps1', # Shell
+ '.sql', '.graphql', '.gql', # Query languages
+ '.yaml', '.yml', '.toml', '.json', '.xml', # Config
+ '.md', '.rst', '.txt', # Docs
+ '.zig', '.nim', '.d', '.v', # Modern systems langs
+ '.ex', '.exs', '.erl', '.hrl', # Erlang/Elixir
+ '.clj', '.cljs', '.cljc', '.edn', # Clojure
+ '.hs', '.lhs', # Haskell
+ '.ml', '.mli', '.re', '.rei', # OCaml/ReasonML
+ '.lisp', '.cl', '.el', '.scm', '.rkt', # Lisps
+ '.f90', '.f95', '.f03', '.for', # Fortran
+ '.asm', '.s', # Assembly
+ '.cob', '.cbl', # COBOL
+ '.pro', # Prolog
+ '.tcl', # Tcl
+ '.dart', # Dart
+ '.raku', '.p6', # Raku
+ '.cr', # Crystal
+ '.vue', '.svelte', # Frontend frameworks
+ '.tf', '.hcl', # Terraform
+ '.dockerfile', '.makefile', # Build files
+}
+
+# Font file extensions
+FONT_EXTENSIONS = {'.woff', '.woff2', '.ttf', '.otf', '.eot', '.sfnt'}
+
+# Style file extensions
+STYLE_EXTENSIONS = {'.css', '.scss', '.sass', '.less', '.styl'}
+
# MIME types by media type
IMAGE_MIME_PREFIXES = ('image/',)
VIDEO_MIME_PREFIXES = ('video/',)
AUDIO_MIME_PREFIXES = ('audio/',)
+CODE_MIME_TYPES = {
+ 'text/x-python', 'application/x-python', 'text/x-python-script',
+ 'text/javascript', 'application/javascript', 'application/x-javascript',
+ 'text/typescript', 'application/typescript',
+ 'text/x-rust', 'text/x-go', 'text/x-ruby', 'application/x-ruby',
+ 'text/x-java-source', 'text/x-kotlin', 'text/x-scala',
+ 'text/x-c', 'text/x-c++', 'text/x-csrc', 'text/x-c++src',
+ 'text/x-csharp', 'text/x-fsharp',
+ 'text/x-swift', 'text/x-objective-c',
+ 'text/x-lua', 'text/x-r', 'text/x-julia',
+ 'text/x-shellscript', 'application/x-sh', 'text/x-bash',
+ 'application/sql', 'application/graphql',
+ 'application/json', 'application/xml', 'text/xml',
+ 'text/yaml', 'application/x-yaml', 'text/x-yaml',
+ 'text/markdown', 'text/x-markdown',
+ 'text/plain', # Often used for code
+}
+FONT_MIME_TYPES = {
+ 'font/woff', 'font/woff2', 'font/ttf', 'font/otf', 'font/sfnt',
+ 'application/font-woff', 'application/font-woff2',
+ 'application/x-font-ttf', 'application/x-font-otf',
+ 'application/vnd.ms-fontobject',
+}
+STYLE_MIME_TYPES = {'text/css', 'text/x-scss', 'text/x-sass', 'text/x-less'}
@dataclass
@@ -376,13 +438,14 @@ def get_media_type_from_extension(url: str) -> Optional[str]:
Determine media type from URL extension.
Returns:
- 'image', 'video', 'audio', or None
+ 'image', 'video', 'audio', 'code', 'font', 'style', or None
"""
parsed = Uri(url)
if not parsed.path:
return None
path = parsed.path.lower()
+ # Check for extension match
for ext in IMAGE_EXTENSIONS:
if path.endswith(ext):
return 'image'
@@ -392,6 +455,15 @@ def get_media_type_from_extension(url: str) -> Optional[str]:
for ext in AUDIO_EXTENSIONS:
if path.endswith(ext):
return 'audio'
+ for ext in CODE_EXTENSIONS:
+ if path.endswith(ext):
+ return 'code'
+ for ext in FONT_EXTENSIONS:
+ if path.endswith(ext):
+ return 'font'
+ for ext in STYLE_EXTENSIONS:
+ if path.endswith(ext):
+ return 'style'
return None
@@ -400,18 +472,26 @@ def get_media_type_from_mime(mime_type: str) -> Optional[str]:
Determine media type from MIME type.
Returns:
- 'image', 'video', 'audio', or None
+ 'image', 'video', 'audio', 'code', 'font', 'style', or None
"""
if not mime_type:
return None
mime_lower = mime_type.lower()
+ # Check prefixes first
if mime_lower.startswith(IMAGE_MIME_PREFIXES):
return 'image'
if mime_lower.startswith(VIDEO_MIME_PREFIXES):
return 'video'
if mime_lower.startswith(AUDIO_MIME_PREFIXES):
return 'audio'
+ # Check exact matches for code/font/style
+ if mime_lower in CODE_MIME_TYPES:
+ return 'code'
+ if mime_lower in FONT_MIME_TYPES:
+ return 'font'
+ if mime_lower in STYLE_MIME_TYPES:
+ return 'style'
return None
@@ -710,16 +790,48 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
if src:
add_media(src, 'audio', title=title)
- # Extract from pointing to media files
+ # Extract from pointing to media/code/font/style files
for a in soup.find_all('a', href=True):
href = a['href']
media_type = get_media_type_from_extension(href)
if media_type:
if (media_type == 'image' and collect_images) or \
(media_type == 'video' and collect_videos) or \
- (media_type == 'audio' and collect_audio):
+ (media_type == 'audio' and collect_audio) or \
+ media_type in ('code', 'font', 'style'): # Always collect code/font/style
add_media(href, media_type, alt_text=a.get_text(strip=True)[:100])
+ # Extract from tags for stylesheets and fonts
+ for link in soup.find_all('link', href=True):
+ href = link.get('href')
+ rel = link.get('rel', [])
+ as_attr = link.get('as', '')
+
+ if 'stylesheet' in rel:
+ add_media(href, 'style', alt_text='stylesheet')
+ elif 'preload' in rel and as_attr == 'font':
+ add_media(href, 'font', alt_text='preload font')
+ elif 'preload' in rel and as_attr == 'style':
+ add_media(href, 'style', alt_text='preload style')
+ else:
+ # Check by extension
+ media_type = get_media_type_from_extension(href)
+ if media_type in ('font', 'style'):
+ add_media(href, media_type, alt_text=f'link {media_type}')
+
+ # Extract from
-
-