update

app.py

@@ -6,7 +6,9 @@ import pandas as pd
 import spaces
 from langchain_openai import ChatOpenAI
 from langchain.agents import create_agent
+from langchain.agents.middleware import ToolCallLimitMiddleware
 from langchain_tavily import TavilySearch
+from langchain_core.messages import SystemMessage
 
 # (Keep Constants as is)
 # --- Constants ---
@@ -25,69 +27,423 @@ def zerogpu_function():
 #         fixed_answer = "This is a default answer."
 #         print(f"Agent returning fixed answer: {fixed_answer}")
 #         return fixed_answer
+from datasets import load_dataset
+from huggingface_hub import hf_hub_download
+from langchain.tools import tool
+from pathlib import Path
+import base64
+from langfuse import get_client
+from langfuse.langchain import CallbackHandler
+from openai import OpenAI
+langfuse = get_client()
+langfuse_handler = CallbackHandler()
+gaia_dataset = load_dataset(
+    "gaia-benchmark/GAIA",
+    "2023_all",
+    split="validation",
+)
+
+task_map = {
+    item["task_id"]: item
+    for item in gaia_dataset
+}
+omni_client = OpenAI(
+    api_key=os.environ["DASHSCOPE_API_KEY"],
+    base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
+)
+@tool
+def read_text_file(file_path: str) -> str:
+    """Read a local text or Python file and return its contents."""
+
+    with open(file_path, "r", encoding="utf-8") as f:
+        return f.read()
+    
+@tool
+def download_attachment(task_id: str, file_name: str) -> str:
+    """
+    Download the attachment associated with a GAIA task.
+
+    Use this tool when the question has an attachment and you need
+    the local file path in order to inspect the file.
+
+    Args:
+        task_id: The GAIA task ID.
+        file_name: The name of the file to download.
+    Returns:
+        The local path of the downloaded attachment.
+    """
+
+    item = task_map.get(task_id)
+
+    if item is None:
+        return f"ERROR: task_id {task_id} was not found. Do not call this tool again."
+
+    if not file_name or not file_name.strip():
+        return "ERROR: This task has no attachment. Do not call this tool again."
+
+    file_name = item.get("file_name")
+    repo_file_path = item.get("file_path")
+
+    if not file_name or not repo_file_path:
+        return "ERROR: This task has no attachment. Do not call this tool again."
+
+    local_path = hf_hub_download(
+        repo_id="gaia-benchmark/GAIA",
+        repo_type="dataset",
+        filename=repo_file_path,
+    )
+
+    return local_path
+
+
+from pypdf import PdfReader
+
+
+@tool
+def read_pdf(file_path: str) -> str:
+    """
+    Read a local PDF file and return its extracted text.
+
+    Use this tool for .pdf attachments after download_attachment
+    has returned the local file path.
+    """
+
+    reader = PdfReader(file_path)
+
+    pages = []
+
+    for page in reader.pages:
+        text = page.extract_text()
+
+        if text:
+            pages.append(text)
+
+    return "\n\n".join(pages)
+
+@tool
+def read_excel(file_path: str) -> str:
+    """Read an Excel spreadsheet and return its contents."""
+
+    import pandas as pd
+
+    df = pd.read_excel(file_path)
+
+    return df.to_string()
+
+@tool
+def analyze_media(file_path: str) -> str:
+    """
+    Inspect an image or audio attachment and return the factual
+    information contained in it.
+
+    This tool is for perception only.
+
+    For images:
+    - Describe visible objects, text, labels, numbers, positions,
+      tables, diagrams, or other observable details accurately.
+    - Preserve exact text and numbers when possible.
+    - Do NOT solve the user's question.
+    - Do NOT infer the final answer.
+    - Do NOT perform domain reasoning beyond what is necessary
+      to describe the media.
+
+    For audio:
+    - Transcribe the spoken content accurately.
+    - Identify clearly observable speakers or sound events when useful.
+    - Do NOT solve the user's question.
+    - Do NOT infer the final answer.
+
+    Args:
+        file_path: Local path of the media file.
+
+    Returns:
+        A factual description or transcription of the media.
+    """
+    path = Path(file_path)
+
+    if not path.exists():
+        return f"ERROR: File does not exist: {file_path}"
+
+    suffix = path.suffix.lower()
+
+    # 读取文件 → Base64
+    with open(path, "rb") as f:
+        file_base64 = base64.b64encode(f.read()).decode("utf-8")
+
+    # ---------- 图片 ----------
+    if suffix in {".png", ".jpg", ".jpeg"}:
+        media_prompt = """
+                Inspect this image carefully.
+        
+                Return only the factual information that is directly observable
+                in the image.
+        
+                Preserve exact text, labels, numbers, symbols, spatial relationships,
+                and positions when relevant.
+        
+                Do not answer any external question.
+                Do not solve the task.
+                Do not infer a final answer.
+                Your job is only to convert the visual information into an accurate
+                text representation for another reasoning agent.
+                """
+        mime_type = {
+            ".png": "image/png",
+            ".jpg": "image/jpeg",
+            ".jpeg": "image/jpeg",
+        }[suffix]
+
+        content = [
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": f"data:{mime_type};base64,{file_base64}"
+                },
+            },
+            {
+                "type": "text",
+                "text": media_prompt.strip(),
+            },
+        ]
+
+    # ---------- 音频 ----------
+    elif suffix in {".mp3", ".wav"}:
+
+        audio_format = suffix[1:]   # ".mp3" -> "mp3"
+        media_prompt = """
+        Listen to this audio carefully.
+
+        Produce an accurate transcription of the spoken content.
+        Preserve names, numbers, dates, and other important details.
+
+        Do not answer questions about the audio.
+        Do not solve any task.
+        Your job is only to convert the audio information into text for
+        another reasoning agent.
+        """
+        content = [
+            {
+                "type": "input_audio",
+                "input_audio": {
+                    "data": f"data:;base64,{file_base64}",
+                    "format": audio_format,
+                },
+            },
+            {
+                "type": "text",
+                "text": media_prompt.strip(),
+            },
+        ]
+
+    else:
+        return f"ERROR: Unsupported media type: {suffix}"
+
+    # ---------- 调 Qwen Omni ----------
+    completion = omni_client.chat.completions.create(
+        model="qwen3.5-omni-flash",
+        messages=[
+            {
+                "role": "user",
+                "content": content,
+            }
+        ],
+        modalities=["text"],
+        stream=True,
+        timeout=20,
+    )
+
+    # Omni 是 streaming，把文字拼起来
+    result = ""
+
+    for chunk in completion:
+        if (
+            chunk.choices
+            and chunk.choices[0].delta.content
+        ):
+            result += chunk.choices[0].delta.content
+
+    return result.strip()
+
 search_tool = TavilySearch(
-    max_results=5,
+    max_results=3,
     topic="general",
-    search_depth="advanced",
+    search_depth="basic",
+    # include_domains=["wikipedia.org"],
 
 )
 model = ChatOpenAI(
-    model="qwen-plus",
+    model="qwen3.5-plus",
     api_key=os.environ["DASHSCOPE_API_KEY"],
     base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
     temperature=0,
+    extra_body={"enable_thinking": False},
 )
 
+from pydantic import BaseModel, Field
+from langchain.agents.structured_output import ToolStrategy
+
+
+class FinalAnswer(BaseModel):
+    answer: str = Field(
+        description=(
+            "Only the exact final answer requested by the question. "
+            "No reasoning, no explanation, no prefix such as "
+            "'Final answer:', and no extra commentary."
+        )
+    )
+
 class BasicAgent:
     def __init__(self):
         print("BasicAgent initialized.")
 
         agent_model = model
+        self.search_calls = 0
+
+        @tool("web_search")
+        def limited_search_tool(query: str) -> str:
+            """Search Wikipedia for facts needed to answer the current question."""
+            if self.search_calls >= 3:
+                return (
+                    "SEARCH_LIMIT_REACHED: You have already used all three allowed "
+                    "web searches for this question. Answer using the information "
+                    "already available."
+                )
+
+            self.search_calls += 1
+            return str(search_tool.invoke({"query": query}))
+
+        self.limited_search_tool = limited_search_tool
 
         self.agent = create_agent(
             model=agent_model,
-            tools=[search_tool],
+            tools=[
+                download_attachment,
+                read_text_file,
+                analyze_media,
+                read_excel,
+                read_pdf,
+                self.limited_search_tool,
+            ],
+            middleware=[
+                ToolCallLimitMiddleware(
+                    tool_name="web_search",
+                    run_limit=3,
+                    exit_behavior="end",
+                )
+            ],
+            response_format=ToolStrategy(FinalAnswer),
             system_prompt="""
-You are solving GAIA benchmark questions.
-
-You have access to a web search tool.
-
-Use web search whenever:
-- The question asks about specific facts that may not be in your knowledge.
-- The question refers to Wikipedia, websites, historical records,
-  specific people, articles, dates, publications, or other information
-  that should be verified.
-- You are uncertain about a factual answer.
-
-You still cannot access files or attachments.
-You do not have Python or a calculator.
-
-When searching:
-- Search multiple times if necessary.
-- Use different search queries when the first search is insufficient.
-- Carefully reason over the search results before answering.
-
-IMPORTANT:
-- Return only the final answer.
-- Do not explain your reasoning.
-- Do not include phrases such as "FINAL ANSWER".
-- Follow the exact output format requested by the question.
+You are an agent solving GAIA benchmark questions.
+
+Follow this decision process strictly.
+
+1. ATTACHMENT CHECK
+
+Only enter the attachment workflow if the user message exactly includes "There is an attachment for this question.".
+
+- Use download_attachment with the provided task_id.
+- After obtaining the local file path, choose the appropriate tool:
+  - Text or source-code file -> read_text_file
+  - Excel file -> read_excel
+  - PDF file -> read_pdf
+  - Image or audio file -> analyze_media
+- Use the information returned by the file tool to answer the original question.
+- Do not invent or guess the contents of the attachment.
+- Inspect the attachment before answering if the question depends on it.
+
+2. NO ATTACHMENT
+
+If the user message does NOT explicitly state that an attachment is present:
+
+- Do NOT call download_attachment.
+- Do NOT call read_text_file.
+- Do NOT call read_excel.
+- Do NOT call read_pdf.
+- Do NOT call analyze_media.
+
+Then determine whether the question requires web search or external information.
+
+Use web_search only when external or up-to-date information is necessary.
+Prefer answering from the available attachment or your own knowledge.
+Use at most three web searches per question, and use fewer when possible.
+
+Make each query precise and Wikipedia-focused.
+Only make a follow-up search when the earlier Wikipedia results are insufficient.
+Do not repeat equivalent or overly broad searches.
+
+
+
+3. FINAL ANSWER FORMAT
+
+Return only the exact final answer requested by the question.
+
+Do not include explanations.
+Do not include reasoning.
+Do not include prefixes such as "Final answer:".
+Follow the exact answer format requested by the question.
+
 """
         )
 
-    def __call__(self, question: str) -> str:
-        print(f"Agent received question: {question[:100]}...")
+    def __call__(
+    self,
+    question: str,
+    task_id: str = "",
+    file_name: str = ""
+    ) -> str:
+        self.search_calls = 0
+
+        content = question
+        if file_name:
+            content += f"""
+
+        There is an attachment for this question.
+        File name: {file_name}
+        Task ID: {task_id}
+
+        Use an appropriate file tool if you need to inspect the attachment.
+        """
+        print(f"Agent received question: {content[:100]}...")
 
         result = self.agent.invoke({
             "messages": [
                 {
                     "role": "user",
-                    "content": question
+                    "content": content
                 }
-            ]
-        })
+            ],
+        },
+        config={
+        "callbacks": [langfuse_handler]
+        }
+        )
 
-        answer = result["messages"][-1].content
+        structured_response = result.get("structured_response")
+        if structured_response is None:
+            print("Web-search limit reached; generating a final answer without tools.")
+            final_message = model.invoke(
+                [
+                    SystemMessage(
+                        content=(
+                            "The research phase is complete because the web-search "
+                            "limit was reached. Do not call tools or browse. Answer "
+                            "the original question using the search results already in "
+                            "this conversation and your own knowledge.\n\n"
+                            "FINAL ANSWER FORMAT — mandatory:\n"
+                            "- Return only the exact final answer requested by the original question.\n"
+                            "- Do not include reasoning, explanations, sources, or prefixes.\n"
+                            "- Follow the exact requested format, including units, dates, "
+                            "capitalization, ordering, and number of items."
+                        )
+                    ),
+                    *result["messages"],
+                ]
+            )
+            answer = str(final_message.content).strip()
+            print(f"Agent answer after search limit: {answer}")
+            return answer or "GIVE UP"
+
+        answer = structured_response.answer.strip()
 
         print(f"Agent answer: {answer}")
 
@@ -99,7 +455,6 @@ IMPORTANT:
 
 
 
-
 def run_and_submit_all( profile: gr.OAuthProfile | None):
     """
     Fetches all questions, runs the BasicAgent on them, submits all answers,
@@ -161,7 +516,7 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
             print(f"Skipping item with missing task_id or question: {item}")
             continue
         try:
-            submitted_answer = agent(question_text)
+            submitted_answer = agent(question_text, task_id=task_id, file_name=item.get("file_name", ""))
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
         except Exception as e:
@@ -274,4 +629,4 @@ if __name__ == "__main__":
     print("-"*(60 + len(" App Starting ")) + "\n")
 
     print("Launching Gradio Interface for Basic Agent Evaluation...")
-    demo.launch(debug=True, share=False)
+    demo.launch(debug=True, share=False)

try_download.py

@@ -0,0 +1,26 @@
+from datasets import load_dataset
+from huggingface_hub import hf_hub_download
+
+task_id = "cca530fc-4052-43b2-b130-b30968d8aa44"
+
+dataset = load_dataset(
+    "gaia-benchmark/GAIA",
+    "2023_level1",
+    split="validation",
+    trust_remote_code=True,
+)
+
+for item in dataset:
+    if item["task_id"] == task_id:
+        print("file_name:", item["file_name"])
+        print("file_path:", item["file_path"])
+
+        file_path = hf_hub_download(
+            repo_id="gaia-benchmark/GAIA",
+            repo_type="dataset",
+            filename=item["file_path"],
+            local_dir="attachments",
+        )
+
+        print("下载到:", file_path)
+        break