Spaces:
Running
Running
Fix
Browse files
app.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
import os
|
| 2 |
import io
|
| 3 |
import json
|
|
@@ -325,16 +326,45 @@ def format_tool_log(tool_name, reason, meta, output, style="A"):
|
|
| 325 |
"detailed": format_tool_log(tool_name, reason, meta, output, style="B"),
|
| 326 |
}
|
| 327 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 328 |
with gr.Blocks(css=".gradio-container {background:#f7fafc}") as demo:
|
| 329 |
gr.Markdown("# Accessibility Voice Agent — MCP Tools")
|
| 330 |
|
| 331 |
with gr.Row():
|
| 332 |
with gr.Column(scale=3):
|
| 333 |
-
|
|
|
|
| 334 |
user_input = gr.Textbox(placeholder="Type or press the microphone to speak...", show_label=False)
|
| 335 |
|
| 336 |
with gr.Row():
|
| 337 |
-
|
|
|
|
| 338 |
send_btn = gr.Button("Send")
|
| 339 |
|
| 340 |
with gr.Accordion("Advanced / Tools", open=False):
|
|
@@ -355,6 +385,8 @@ with gr.Blocks(css=".gradio-container {background:#f7fafc}") as demo:
|
|
| 355 |
# Callbacks
|
| 356 |
def on_send_text(text, chat_history, mic_file, style):
|
| 357 |
tools_entries = []
|
|
|
|
|
|
|
| 358 |
if mic_file:
|
| 359 |
# transcribe audio
|
| 360 |
tr = transcribe_audio_tool(mic_file)
|
|
@@ -362,9 +394,10 @@ with gr.Blocks(css=".gradio-container {background:#f7fafc}") as demo:
|
|
| 362 |
log = format_tool_log("transcribe_audio", "User provided microphone audio", tr.meta or {}, tr.content, style)
|
| 363 |
tools_entries.append(log)
|
| 364 |
else:
|
| 365 |
-
user_text = text
|
| 366 |
-
|
| 367 |
-
|
|
|
|
| 368 |
|
| 369 |
# demo assistant behavior
|
| 370 |
if user_text and user_text.strip().lower().startswith("describe image:"):
|
|
@@ -382,7 +415,8 @@ with gr.Blocks(css=".gradio-container {background:#f7fafc}") as demo:
|
|
| 382 |
else:
|
| 383 |
assistant = "I heard: " + (user_text or "(empty)")
|
| 384 |
|
| 385 |
-
|
|
|
|
| 386 |
|
| 387 |
# update tools panel content
|
| 388 |
panel_html = ''
|
|
@@ -393,7 +427,10 @@ with gr.Blocks(css=".gradio-container {background:#f7fafc}") as demo:
|
|
| 393 |
else:
|
| 394 |
for e in tools_entries:
|
| 395 |
panel_html += f"<pre style='background:#f1f5f9;border-radius:6px;padding:8px;margin-bottom:8px;'>{e}</pre>"
|
| 396 |
-
|
|
|
|
|
|
|
|
|
|
| 397 |
|
| 398 |
send_btn.click(on_send_text, inputs=[user_input, chatbox, mic, log_style], outputs=[chatbox, tools_log, tools_panel])
|
| 399 |
|
|
@@ -415,14 +452,30 @@ with gr.Blocks(css=".gradio-container {background:#f7fafc}") as demo:
|
|
| 415 |
|
| 416 |
def on_describe_image(file_obj, style):
|
| 417 |
if not file_obj:
|
| 418 |
-
return "No file uploaded"
|
| 419 |
-
# file_obj may be
|
| 420 |
-
path = getattr(file_obj, 'name',
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 421 |
res = describe_image_tool(path)
|
| 422 |
log = format_tool_log("describe_image", "User uploaded an image for description", res.meta or {}, res.content, style)
|
| 423 |
panel_html = f"<pre style='background:#ecfdf5;padding:8px;border-radius:6px;'>{log}</pre>"
|
| 424 |
-
|
| 425 |
-
|
|
|
|
|
|
|
| 426 |
|
| 427 |
img_btn.click(on_describe_image, inputs=[img_upload, log_style], outputs=[chatbox, tools_panel])
|
| 428 |
|
|
|
|
| 1 |
+
|
| 2 |
import os
|
| 3 |
import io
|
| 4 |
import json
|
|
|
|
| 326 |
"detailed": format_tool_log(tool_name, reason, meta, output, style="B"),
|
| 327 |
}
|
| 328 |
|
| 329 |
+
# Conversion helpers for chat history between 'messages' (gradio new) and tuple list used in logic
|
| 330 |
+
def messages_to_tuples(messages):
|
| 331 |
+
# messages is a list of dicts {"role": "user"/"assistant", "content": "..."}
|
| 332 |
+
tuples = []
|
| 333 |
+
if not messages:
|
| 334 |
+
return tuples
|
| 335 |
+
for m in messages:
|
| 336 |
+
if isinstance(m, dict):
|
| 337 |
+
role = m.get("role", "user")
|
| 338 |
+
content = m.get("content", "")
|
| 339 |
+
tuples.append((content, "")) if role == "user" else tuples.append(("", content))
|
| 340 |
+
elif isinstance(m, (list, tuple)) and len(m) == 2:
|
| 341 |
+
tuples.append((m[0], m[1]))
|
| 342 |
+
else:
|
| 343 |
+
# fallback: treat as assistant reply
|
| 344 |
+
tuples.append(("", str(m)))
|
| 345 |
+
return tuples
|
| 346 |
+
|
| 347 |
+
def tuples_to_messages(tuples):
|
| 348 |
+
messages = []
|
| 349 |
+
for user_text, assistant_text in tuples:
|
| 350 |
+
if user_text:
|
| 351 |
+
messages.append({"role":"user","content":user_text})
|
| 352 |
+
if assistant_text:
|
| 353 |
+
messages.append({"role":"assistant","content":assistant_text})
|
| 354 |
+
return messages
|
| 355 |
+
|
| 356 |
with gr.Blocks(css=".gradio-container {background:#f7fafc}") as demo:
|
| 357 |
gr.Markdown("# Accessibility Voice Agent — MCP Tools")
|
| 358 |
|
| 359 |
with gr.Row():
|
| 360 |
with gr.Column(scale=3):
|
| 361 |
+
# Set type='messages' to avoid the deprecation warning, and convert inside handlers.
|
| 362 |
+
chatbox = gr.Chatbot(label="Assistant", elem_id="chatbox", type="messages")
|
| 363 |
user_input = gr.Textbox(placeholder="Type or press the microphone to speak...", show_label=False)
|
| 364 |
|
| 365 |
with gr.Row():
|
| 366 |
+
# Some gradio versions don't accept 'source' kw; remove it to be broadly compatible.
|
| 367 |
+
mic = gr.Audio(type="filepath", label="Record voice (press to record)")
|
| 368 |
send_btn = gr.Button("Send")
|
| 369 |
|
| 370 |
with gr.Accordion("Advanced / Tools", open=False):
|
|
|
|
| 385 |
# Callbacks
|
| 386 |
def on_send_text(text, chat_history, mic_file, style):
|
| 387 |
tools_entries = []
|
| 388 |
+
# convert incoming chat_history (messages) into tuples for internal logic
|
| 389 |
+
tuples = messages_to_tuples(chat_history)
|
| 390 |
if mic_file:
|
| 391 |
# transcribe audio
|
| 392 |
tr = transcribe_audio_tool(mic_file)
|
|
|
|
| 394 |
log = format_tool_log("transcribe_audio", "User provided microphone audio", tr.meta or {}, tr.content, style)
|
| 395 |
tools_entries.append(log)
|
| 396 |
else:
|
| 397 |
+
user_text = text or ""
|
| 398 |
+
|
| 399 |
+
# Append user message to tuples and placeholder assistant
|
| 400 |
+
tuples.append((user_text, "..."))
|
| 401 |
|
| 402 |
# demo assistant behavior
|
| 403 |
if user_text and user_text.strip().lower().startswith("describe image:"):
|
|
|
|
| 415 |
else:
|
| 416 |
assistant = "I heard: " + (user_text or "(empty)")
|
| 417 |
|
| 418 |
+
# replace placeholder assistant
|
| 419 |
+
tuples[-1] = (tuples[-1][0], assistant)
|
| 420 |
|
| 421 |
# update tools panel content
|
| 422 |
panel_html = ''
|
|
|
|
| 427 |
else:
|
| 428 |
for e in tools_entries:
|
| 429 |
panel_html += f"<pre style='background:#f1f5f9;border-radius:6px;padding:8px;margin-bottom:8px;'>{e}</pre>"
|
| 430 |
+
|
| 431 |
+
# convert back to messages for gr.Chatbot
|
| 432 |
+
new_messages = tuples_to_messages(tuples)
|
| 433 |
+
return new_messages, gr.update(value="\n".join(tools_entries) or "Ready."), gr.update(value=panel_html)
|
| 434 |
|
| 435 |
send_btn.click(on_send_text, inputs=[user_input, chatbox, mic, log_style], outputs=[chatbox, tools_log, tools_panel])
|
| 436 |
|
|
|
|
| 452 |
|
| 453 |
def on_describe_image(file_obj, style):
|
| 454 |
if not file_obj:
|
| 455 |
+
return [], gr.update(value="No file uploaded")
|
| 456 |
+
# file_obj may be an UploadFile-like object; get path or save to tmp file
|
| 457 |
+
path = getattr(file_obj, 'name', None)
|
| 458 |
+
# If it's a temporary file dict (from gr.File), it might be a dict with 'name' and 'tmp_path'
|
| 459 |
+
if isinstance(file_obj, dict) and 'tmp_path' in file_obj:
|
| 460 |
+
path = file_obj['tmp_path']
|
| 461 |
+
if not path:
|
| 462 |
+
# try to save bytes
|
| 463 |
+
try:
|
| 464 |
+
contents = file_obj.read()
|
| 465 |
+
tmp_path = "/tmp/gr_uploaded_image.jpg"
|
| 466 |
+
with open(tmp_path, "wb") as f:
|
| 467 |
+
f.write(contents)
|
| 468 |
+
path = tmp_path
|
| 469 |
+
except Exception as e:
|
| 470 |
+
return [], gr.update(value=f"Failed to read uploaded file: {e}")
|
| 471 |
+
|
| 472 |
res = describe_image_tool(path)
|
| 473 |
log = format_tool_log("describe_image", "User uploaded an image for description", res.meta or {}, res.content, style)
|
| 474 |
panel_html = f"<pre style='background:#ecfdf5;padding:8px;border-radius:6px;'>{log}</pre>"
|
| 475 |
+
|
| 476 |
+
# Return as messages for chatbox
|
| 477 |
+
messages = [{"role":"user","content":"<image uploaded>"}, {"role":"assistant","content":res.content}]
|
| 478 |
+
return messages, gr.update(value=panel_html)
|
| 479 |
|
| 480 |
img_btn.click(on_describe_image, inputs=[img_upload, log_style], outputs=[chatbox, tools_panel])
|
| 481 |
|