From 77c2ef1d83c0f97b8a89c93aff8dedd5c57c1740 Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Thu, 4 Jul 2024 10:21:37 -0400 Subject: [PATCH 1/5] messing around with keyword search working but ugly! modified: app.py new file: templates/base.html modified: templates/chat.html new file: templates/search.html --- app.py | 71 ++++++++++++++ templates/base.html | 217 ++++++++++++++++++++++++++++++++++++++++++ templates/chat.html | 191 +------------------------------------ templates/search.html | 24 +++++ 4 files changed, 317 insertions(+), 186 deletions(-) create mode 100644 templates/base.html create mode 100644 templates/search.html diff --git a/app.py b/app.py index 2538c1b..5b37dea 100644 --- a/app.py +++ b/app.py @@ -138,6 +138,77 @@ def chat(room_name): ) +@app.route("/search") +def search_page(): + keywords = request.args.get("keywords", "") + if not keywords: + return render_template("search_results.html", results=[], error="Keywords are required") + + # Call the function to search messages + search_results = search_messages(keywords) + + return render_template("search.html", results=search_results, error=None) + + +def search_messages(keywords): + search_results = {} + + # Split the keywords by spaces + keyword_list = keywords.lower().split() + + # Search for messages containing any of the keywords + messages = Message.query.filter( + db.or_(*[Message.content.ilike(f"%{keyword}%") for keyword in keyword_list]) + ).all() + + for message in messages: + room = Room.query.get(message.room_id) + if room: + # Calculate the score based on the number of occurrences of all keywords + score = sum(message.content.lower().count(keyword) for keyword in keyword_list) + + # Extract snippets with context around each occurrence of the keywords + snippets = [] + content_lower = message.content.lower() + + for keyword in keyword_list: + start_index = 0 + while start_index < len(content_lower): + start_index = content_lower.find(keyword, start_index) + if start_index == -1: + break + + # Calculate the snippet range + snippet_start = max(0, start_index - 25) + snippet_end = min(len(message.content), start_index + len(keyword) + 25) + snippet = message.content[snippet_start:snippet_end] + + snippets.append(snippet) + start_index += len(keyword) + + # Join all snippets for this message + snippet_text = " ... ".join(snippets) + + if room.id not in search_results: + search_results[room.id] = { + "room_id": room.id, + "room_name": room.name, + "room_title": room.title, + "snippets": [], + "username": message.username, + "score": 0 + } + + search_results[room.id]["snippets"].append(snippet_text) + search_results[room.id]["score"] += score + + # Convert the dictionary to a list and sort results by score in descending order + search_results_list = list(search_results.values()) + search_results_list.sort(key=lambda x: x["score"], reverse=True) + + return search_results_list + + @socketio.on("join") def on_join(data): room_name = data["room_name"] diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..bc8b63e --- /dev/null +++ b/templates/base.html @@ -0,0 +1,217 @@ + + + + + + {% block title %}Chatroom{% endblock %} + + + + + + + + + + + + + + + + + + + + + + + + 🚀 docs for interacting with language models & other commands + +
+
+ +
+
+ +
+
+ +
+ {% block content %}{% endblock %} +
+ + + + diff --git a/templates/chat.html b/templates/chat.html index 9da7eb6..9e9eced 100644 --- a/templates/chat.html +++ b/templates/chat.html @@ -1,182 +1,8 @@ - - - - - - Chatroom +{% extends "base.html" %} - - - - - - - - - - - - - - - - - - - - 🚀 docs for interacting with language models & other commands -
-
- -
+{% block title %}Chatroom{% endblock %} +{% block content %}
@@ -186,12 +12,7 @@
-
- - - + +{% endblock %} diff --git a/templates/search.html b/templates/search.html new file mode 100644 index 0000000..c4d3183 --- /dev/null +++ b/templates/search.html @@ -0,0 +1,24 @@ +{% extends "base.html" %} + +{% block title %}Search Results{% endblock %} + +{% block content %} +
+ {% if error %} +
{{ error }}
+ {% elif results %} + {% for result in results %} +
+ + {{ result.room_name }} ({{ result.room_title or "No title" }})
+ Snippets: {{ result.snippets | join(' ... ') }}
+ Score: {{ result.score }} + +
+
+ {% endfor %} + {% else %} +
No results found.
+ {% endif %} +
+{% endblock %} From 557782185517e44a614fdebcdbfe4776f5a8771b Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Thu, 4 Jul 2024 10:57:30 -0400 Subject: [PATCH 2/5] save username when searching I CAN program offline without an LLM. ; ) modified: app.py modified: templates/base.html modified: templates/search.html --- app.py | 56 +++++++++++++++++++++++++------------------ templates/base.html | 4 +++- templates/search.html | 14 +++++++++++ 3 files changed, 50 insertions(+), 24 deletions(-) diff --git a/app.py b/app.py index 5b37dea..5ae36d6 100644 --- a/app.py +++ b/app.py @@ -1,8 +1,9 @@ -#import eventlet -#eventlet.monkey_patch() +# import eventlet +# eventlet.monkey_patch() import gevent from gevent import monkey + monkey.patch_all() @@ -28,7 +29,7 @@ app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False db = SQLAlchemy(app) -#socketio = SocketIO(app, async_mode="eventlet") +# socketio = SocketIO(app, async_mode="eventlet") socketio = SocketIO(app, async_mode="gevent") # Global dictionary to keep track of cancellation requests @@ -141,13 +142,18 @@ def chat(room_name): @app.route("/search") def search_page(): keywords = request.args.get("keywords", "") + username = request.args.get("username", "guest") if not keywords: - return render_template("search_results.html", results=[], error="Keywords are required") + return render_template( + "search.html", results=[], username=username, error="Keywords are required" + ) # Call the function to search messages search_results = search_messages(keywords) - return render_template("search.html", results=search_results, error=None) + return render_template( + "search.html", results=search_results, username=username, error=None + ) def search_messages(keywords): @@ -165,7 +171,9 @@ def search_messages(keywords): room = Room.query.get(message.room_id) if room: # Calculate the score based on the number of occurrences of all keywords - score = sum(message.content.lower().count(keyword) for keyword in keyword_list) + score = sum( + message.content.lower().count(keyword) for keyword in keyword_list + ) # Extract snippets with context around each occurrence of the keywords snippets = [] @@ -180,7 +188,9 @@ def search_messages(keywords): # Calculate the snippet range snippet_start = max(0, start_index - 25) - snippet_end = min(len(message.content), start_index + len(keyword) + 25) + snippet_end = min( + len(message.content), start_index + len(keyword) + 25 + ) snippet = message.content[snippet_start:snippet_end] snippets.append(snippet) @@ -196,7 +206,7 @@ def search_messages(keywords): "room_title": room.title, "snippets": [], "username": message.username, - "score": 0 + "score": 0, } search_results[room.id]["snippets"].append(snippet_text) @@ -533,32 +543,32 @@ def group_consecutive_roles(messages): return [] grouped_messages = [] - current_role = messages[0]['role'] + current_role = messages[0]["role"] current_content = [] for message in messages: - if message['role'] == current_role: - current_content.append(message['content']) + if message["role"] == current_role: + current_content.append(message["content"]) else: - grouped_messages.append({ - 'role': current_role, - 'content': ' '.join(current_content) - }) - current_role = message['role'] - current_content = [message['content']] + grouped_messages.append( + {"role": current_role, "content": " ".join(current_content)} + ) + current_role = message["role"] + current_content = [message["content"]] # Append the last grouped message - grouped_messages.append({ - 'role': current_role, - 'content': ' '.join(current_content) - }) + grouped_messages.append( + {"role": current_role, "content": " ".join(current_content)} + ) return grouped_messages def chat_claude( - #username, room_name, model_name="anthropic.claude-3-5-sonnet-20240620-v1:0" - username, room_name, model_name="anthropic.claude-3-sonnet-20240229-v1:0" + # username, room_name, model_name="anthropic.claude-3-5-sonnet-20240620-v1:0" + username, + room_name, + model_name="anthropic.claude-3-sonnet-20240229-v1:0", ): with app.app_context(): room = get_room(room_name) diff --git a/templates/base.html b/templates/base.html index bc8b63e..3d9657a 100644 --- a/templates/base.html +++ b/templates/base.html @@ -172,6 +172,7 @@
+
@@ -196,13 +197,14 @@ // Function to perform the search function performSearch() { const keywords = document.getElementById("search-keywords").value; + const username = document.getElementById("username").value; if (!keywords) { alert("Please enter keywords to search."); return; } // Navigate to the search results page with the keywords as a query parameter - window.location.href = `/search?keywords=${encodeURIComponent(keywords)}`; + window.location.href = `/search?keywords=${encodeURIComponent(keywords)}&username=${username}`; } // Add event listener for keyword search the "Enter" key diff --git a/templates/search.html b/templates/search.html index c4d3183..d58c7ac 100644 --- a/templates/search.html +++ b/templates/search.html @@ -3,6 +3,19 @@ {% block title %}Search Results{% endblock %} {% block content %} + +
{% if error %}
{{ error }}
@@ -22,3 +35,4 @@ {% endif %}
{% endblock %} + From 926262cdaa068e734a69cbda1e7c3f0a834f8922 Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Thu, 4 Jul 2024 11:12:21 -0400 Subject: [PATCH 3/5] keep keywords in search form. modified: app.py modified: templates/base.html --- app.py | 12 ++++++++++-- templates/base.html | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 5ae36d6..8061551 100644 --- a/app.py +++ b/app.py @@ -145,14 +145,22 @@ def search_page(): username = request.args.get("username", "guest") if not keywords: return render_template( - "search.html", results=[], username=username, error="Keywords are required" + "search.html", + keywords=keywords, + results=[], + username=username, + error="Keywords are required", ) # Call the function to search messages search_results = search_messages(keywords) return render_template( - "search.html", results=search_results, username=username, error=None + "search.html", + keywords=keywords, + results=search_results, + username=username, + error=None, ) diff --git a/templates/base.html b/templates/base.html index 3d9657a..abf89ed 100644 --- a/templates/base.html +++ b/templates/base.html @@ -171,7 +171,7 @@
- +
From db3c15ea0dd1aa54e85af72679d6d0694e4bbbbc Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Thu, 4 Jul 2024 11:31:38 -0400 Subject: [PATCH 4/5] hide snippets pass room list in search page modified: app.py modified: templates/base.html modified: templates/search.html --- app.py | 5 +++++ templates/base.html | 2 +- templates/search.html | 9 ++++++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index 8061551..81fe43d 100644 --- a/app.py +++ b/app.py @@ -141,11 +141,15 @@ def chat(room_name): @app.route("/search") def search_page(): + # Query all rooms so that newest is first. + rooms = Room.query.order_by(Room.id.desc()).all() + keywords = request.args.get("keywords", "") username = request.args.get("username", "guest") if not keywords: return render_template( "search.html", + rooms=rooms, keywords=keywords, results=[], username=username, @@ -157,6 +161,7 @@ def search_page(): return render_template( "search.html", + rooms=rooms, keywords=keywords, results=search_results, username=username, diff --git a/templates/base.html b/templates/base.html index abf89ed..34c3423 100644 --- a/templates/base.html +++ b/templates/base.html @@ -171,7 +171,7 @@
- +
diff --git a/templates/search.html b/templates/search.html index d58c7ac..e86c76a 100644 --- a/templates/search.html +++ b/templates/search.html @@ -6,13 +6,15 @@ @@ -23,8 +25,9 @@ {% for result in results %}
- {{ result.room_name }} ({{ result.room_title or "No title" }})
- Snippets: {{ result.snippets | join(' ... ') }}
+ {{ result.room_name }}
+ {{ result.room_title or "No title" }}
+ Score: {{ result.score }}
From 57d8e701f2f53376e865f905649d3a8739388701 Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Thu, 4 Jul 2024 12:04:46 -0400 Subject: [PATCH 5/5] remove username and snippet from SERP page --- app.py | 27 --------------------------- templates/search.html | 1 - 2 files changed, 28 deletions(-) diff --git a/app.py b/app.py index 81fe43d..23df7ec 100644 --- a/app.py +++ b/app.py @@ -188,41 +188,14 @@ def search_messages(keywords): message.content.lower().count(keyword) for keyword in keyword_list ) - # Extract snippets with context around each occurrence of the keywords - snippets = [] - content_lower = message.content.lower() - - for keyword in keyword_list: - start_index = 0 - while start_index < len(content_lower): - start_index = content_lower.find(keyword, start_index) - if start_index == -1: - break - - # Calculate the snippet range - snippet_start = max(0, start_index - 25) - snippet_end = min( - len(message.content), start_index + len(keyword) + 25 - ) - snippet = message.content[snippet_start:snippet_end] - - snippets.append(snippet) - start_index += len(keyword) - - # Join all snippets for this message - snippet_text = " ... ".join(snippets) - if room.id not in search_results: search_results[room.id] = { "room_id": room.id, "room_name": room.name, "room_title": room.title, - "snippets": [], - "username": message.username, "score": 0, } - search_results[room.id]["snippets"].append(snippet_text) search_results[room.id]["score"] += score # Convert the dictionary to a list and sort results by score in descending order diff --git a/templates/search.html b/templates/search.html index e86c76a..36db29b 100644 --- a/templates/search.html +++ b/templates/search.html @@ -27,7 +27,6 @@ {{ result.room_name }}
{{ result.room_title or "No title" }}
- Score: {{ result.score }}