Build a Research Agent with AutoGPT (Web Search + Summarize)

# Install AutoGPT
git clone https://github.com/Significant-Gravitas/AutoGPT
cd AutoGPT/autogpts/autogpt

# Install dependencies
poetry install

# Copy and configure environment
cp .env.template .env

# .env file — search configuration
OPENAI_API_KEY=sk-your-openai-key

# Search backend — choose one
GOOGLE_API_KEY=your-google-api-key
CUSTOM_SEARCH_ENGINE_ID=your-search-engine-id

# OR use SerpAPI
SERPAPI_API_KEY=your-serpapi-key

# OR use DuckDuckGo (free but rate-limited)
# No key needed, just set:
SEARCH_BACKEND=duckduckgo

# Browser for reading web pages
USE_WEB_BROWSER=selenium
HEADLESS_BROWSER=True

# Model configuration
SMART_LLM_MODEL=gpt-4o
FAST_LLM_MODEL=gpt-4o-mini

# Output and memory
MEMORY_BACKEND=local
WORKSPACE_BACKEND=local
RESTRICT_TO_WORKSPACE=True

# Cost control
CONTINUOUS_LIMIT=25  # max actions per run

# research_agent.yaml
ai_name: ResearchAgent
ai_role: >
  An autonomous research assistant that searches the web, reads sources,
  and produces structured reports with citations.

ai_goals:
  - >
    Search for "{TOPIC}" using web search. Find at least 5 credible sources
    published in {YEAR_RANGE}. Prefer: academic papers, official documentation,
    major tech publications (TechCrunch, Wired, MIT Technology Review).
  
  - >
    For each source found, browse the URL and extract:
    (1) Key claim or finding, (2) Supporting evidence or data,
    (3) Publication date, (4) Author or organization.
    
  - >
    Write a structured research report to research_report.md containing:
    (1) Executive summary (3-5 sentences),
    (2) Key findings as bullet points with citations,
    (3) Comparison table if multiple items are being compared,
    (4) Conclusion with 2-3 actionable insights.
    
  - >
    Verify that research_report.md exists and contains at least 500 words
    with citations for at least 3 sources.
    
  - >
    Task is COMPLETE when research_report.md is written and verified.
    Do not search for more sources after the file is written.

api_budget: 3.00

from langchain_experimental.autonomous_agents import AutoGPT
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain.vectorstores import FAISS
from langchain.tools import Tool
from langchain_community.tools import DuckDuckGoSearchRun
from langchain.tools.file_management import WriteFileTool, ReadFileTool
from langchain_community.document_loaders import WebBaseLoader
import json

# Initialize components
llm = ChatOpenAI(model="gpt-4o", temperature=0.2, max_tokens=4000)
embeddings = OpenAIEmbeddings()
vectorstore = FAISS.from_texts(["initial"], embeddings)

# Search tool
search = DuckDuckGoSearchRun()

# Web reader tool — fetches and summarizes web pages
def read_webpage(url: str) -> str:
    """Read and extract text content from a URL."""
    try:
        loader = WebBaseLoader(url)
        docs = loader.load()
        if docs:
            # Return first 3000 characters to avoid token overflow
            content = docs[0].page_content[:3000]
            return f"Content from {url}:\n{content}"
        return f"Could not load content from {url}"
    except Exception as e:
        return f"Error reading {url}: {str(e)}"

webpage_tool = Tool(
    name="read_webpage",
    func=read_webpage,
    description="""Read the full content of a webpage. 
    Input: a complete URL starting with http:// or https://
    Output: extracted text content from the page.""",
)

# Citation tracker
citations = []

def add_citation(citation_json: str) -> str:
    """Save a citation. Input: JSON with 'source', 'title', 'finding' keys."""
    try:
        citation = json.loads(citation_json)
        citations.append(citation)
        return f"Citation saved. Total citations: {len(citations)}"
    except:
        return "Citation format error. Use JSON: {\"source\": \"url\", \"title\": \"title\", \"finding\": \"key finding\"}"

citation_tool = Tool(
    name="save_citation",
    func=add_citation,
    description="Save a citation from a source. Use after reading each relevant webpage.",
)

tools = [
    search,
    webpage_tool,
    citation_tool,
    WriteFileTool(),
    ReadFileTool(),
]

# Create the AutoGPT agent
agent = AutoGPT.from_llm_and_tools(
    ai_name="ResearchBot",
    ai_role="""An autonomous research assistant that produces well-cited reports.
    Always search first, then read sources, then write the report.
    Never make up statistics or claims — only use information from sources you have read.""",
    tools=tools,
    llm=llm,
    memory=vectorstore.as_retriever(),
)

agent.chain.verbose = True

# Run the research agent
research_topic = "the impact of vector databases on enterprise AI applications in 2024-2025"

agent.run([
    f"Search for: {research_topic}",
    "Read the top 4-5 most relevant results. Use save_citation for each key finding.",
    "Write a structured research report to vector_db_report.md with: summary, key findings with citations, and conclusion.",
    "Stop after the report is written. Do not continue searching.",
])

print(f"\nCollected {len(citations)} citations")
for i, c in enumerate(citations, 1):
    print(f"{i}. {c.get('title', 'Unknown')} — {c.get('source', '')}")

from langchain.chains import LLMChain
from langchain.prompts import PromptTemplate

summarize_prompt = PromptTemplate(
    input_variables=["content", "topic"],
    template="""
You are a research assistant. Summarize the following content in the context of: {topic}

Content:
{content}

Provide a summary with:
1. Main claim or finding (1-2 sentences)
2. Key supporting data or evidence
3. Relevance to the topic

Summary:"""
)

summarize_chain = LLMChain(
    llm=ChatOpenAI(model="gpt-4o-mini", temperature=0),  # cheaper model for summarization
    prompt=summarize_prompt,
)

def summarize_webpage(url_and_topic: str) -> str:
    """
    Summarize a webpage in context of the research topic.
    Input format: 'URL|||RESEARCH_TOPIC'
    """
    parts = url_and_topic.split("|||")
    if len(parts) != 2:
        return "Format error. Use: URL|||research topic"
    
    url, topic = parts[0].strip(), parts[1].strip()
    
    # Fetch content
    content = read_webpage(url)
    
    # Summarize
    result = summarize_chain.run(content=content, topic=topic)
    return result

summarize_tool = Tool(
    name="summarize_webpage",
    func=summarize_webpage,
    description="""Fetch and summarize a webpage in context of your research topic.
    Input format: 'https://url.com|||your research topic'
    Better than read_webpage for extracting relevant information.""",
)

report_format_goal = """
Write research_report.md with exactly this structure:

# [Topic] Research Brief
*Date: [today's date]*

## Executive Summary
[3-5 sentences summarizing the key findings]

## Key Findings

### [Finding 1 Title]
[2-3 sentences with evidence]
*Source: [URL]*

### [Finding 2 Title]
[2-3 sentences with evidence]
*Source: [URL]*

[Continue for all findings]

## Comparison Table

| Aspect | Option A | Option B | Option C |
|--------|----------|----------|----------|
| [row]  | [value]  | [value]  | [value]  |

## Recommendations
1. [Actionable recommendation]
2. [Actionable recommendation]
3. [Actionable recommendation]

## Sources
- [Title] — [URL] — [Accessed date]
"""

json_format_goal = """
Save research results to research_results.json with this structure:
{
  "topic": "string",
  "date": "YYYY-MM-DD",
  "summary": "string",
  "findings": [
    {
      "title": "string",
      "content": "string",
      "source_url": "string",
      "relevance_score": 1-5
    }
  ],
  "recommendations": ["string"],
  "sources": [{"title": "string", "url": "string"}]
}
"""

research_topics = [
    "Pinecone vector database pricing and performance 2025",
    "Weaviate open source deployment options 2025",
    "Qdrant vs Pinecone performance benchmarks 2025",
]

results = {}

for topic in research_topics:
    print(f"\nResearching: {topic}")
    
    # Fresh agent for each topic to avoid context contamination
    topic_agent = AutoGPT.from_llm_and_tools(
        ai_name="ResearchBot",
        ai_role="Precise research assistant. Only report verified facts from sources.",
        tools=tools,
        llm=llm,
        memory=FAISS.from_texts(["initial"], embeddings).as_retriever(),
    )
    
    safe_filename = topic.replace(" ", "_")[:50]
    
    topic_agent.run([
        f"Search for: {topic}",
        f"Read the top 3 results. Extract key facts.",
        f"Save findings to {safe_filename}.json as a JSON object with 'topic', 'findings', 'sources' keys.",
        "Stop after saving the file.",
    ])
    
    results[topic] = safe_filename

print("\nAll topics researched. Files saved:")
for topic, filename in results.items():
    print(f"  {filename}.json — {topic}")

import autogen

# AutoGPT handles raw research and saves to files
# AutoGen handles synthesis and final report generation

config_list = [{"model": "gpt-4o", "api_key": "YOUR_KEY"}]
llm_config = {"config_list": config_list}

synthesizer = autogen.AssistantAgent(
    name="Synthesizer",
    system_message="""You synthesize research from multiple files into a cohesive report.
    When given a list of JSON research files, read each one and combine the findings.
    Produce a final report that avoids duplication and highlights consensus findings.""",
    llm_config=llm_config,
)

editor = autogen.AssistantAgent(
    name="Editor",
    system_message="""You edit and polish research reports. 
    Check for: logical flow, repetition, unsupported claims, and readability.
    Return a polished version ready for publication.""",
    llm_config=llm_config,
)

user = autogen.UserProxyAgent(
    name="User",
    human_input_mode="NEVER",
    code_execution_config={"work_dir": "workspace"},
)

# After AutoGPT research phase
user.initiate_chat(
    synthesizer,
    message="""Read the research files: pinecone_research.json, weaviate_research.json, 
    qdrant_research.json. Synthesize a comprehensive comparison report.""",
)