refactor: rename tokenizer to token_estimator for clarity
- Rename tokenizer.js to token_estimator.js to reflect its purpose - Update import in translation.js - Add development journal documenting NVIDIA 4090 performance metrics
This commit is contained in:
parent
2845bb16d8
commit
9e6bbe4b19
3 changed files with 21 additions and 1 deletions
20
journal.rst
Normal file
20
journal.rst
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
====================
|
||||
Development Journal
|
||||
====================
|
||||
|
||||
July 2, 2025
|
||||
============
|
||||
|
||||
Token Estimator Implementation
|
||||
-------------------------------
|
||||
|
||||
Built `token_estimator.js` - a simplified BPE-based token counter for translation ETA estimation.
|
||||
|
||||
**Not compatible with tiktoken** - this is estimation-only, ~85% accuracy. Good enough for progress bars, not for exact tokenization.
|
||||
|
||||
**NVIDIA 4090 Performance Metrics:**
|
||||
- Prompt processing: 3,144 tokens/sec
|
||||
- Generation: 121 tokens/sec
|
||||
- Translation estimates: 1K tokens ≈ 11 seconds
|
||||
|
||||
The estimator uses real production metrics from our Hermes machine to provide realistic translation timing.
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
import { sendMessage } from "./chat.js";
|
||||
import { API_KEY } from "./config.js";
|
||||
import { getSelectedModel, getSelectedModelEndpoint } from "./models.js";
|
||||
import { countTokens } from "./tokenizer.js";
|
||||
import { countTokens } from "./token_estimator.js";
|
||||
|
||||
// Languages supported by Hermes 3 model (from Russell's implementation)
|
||||
export const SUPPORTED_LANGUAGES = {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue