LangChain BaseRetriever

pip install langchain langchain-core httpx
export KRAWL_API_KEY=krawl_demo_xxx

curl -X POST https://api.krawl.sh/search \
  -H "X-API-Key: $KRAWL_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"query": "EIP-4844 blob fees", "limit": 10}'

import os, httpx, sys
from langchain_core.callbacks import CallbackManagerForRetrieverRun
from langchain_core.documents import Document
from langchain_core.retrievers import BaseRetriever

class KrawlRetriever(BaseRetriever):
    base_url: str = "https://api.krawl.sh"
    api_key: str = os.environ["KRAWL_API_KEY"]
    k: int = 10
    sources: list[str] | None = None

    def _get_relevant_documents(self, query, *, run_manager: CallbackManagerForRetrieverRun):
        body = {"query": query, "limit": self.k}
        if self.sources:
            body["sources"] = self.sources
        r = httpx.post(
            f"{self.base_url}/search",
            headers={"X-API-Key": self.api_key, "Content-Type": "application/json"},
            json=body,
            timeout=60,
        )
        r.raise_for_status()
        return [
            Document(
                page_content=hit.get("content") or hit.get("snippet") or "",
                metadata={
                    "url": hit.get("url"),
                    "title": hit.get("title"),
                    "source_type": hit.get("source_type"),
                    "score": hit.get("score"),
                },
            )
            for hit in r.json().get("results", [])
        ]

retriever = KrawlRetriever(k=5, sources=["exa", "academic"])
for doc in retriever.invoke(" ".join(sys.argv[1:])):
    print(doc.metadata["url"], doc.metadata["title"])

// LangChain.js: implement BaseRetriever the same way.
import { BaseRetriever } from "@langchain/core/retrievers";
import { Document } from "@langchain/core/documents";

class KrawlRetriever extends BaseRetriever {
  lc_namespace = ["krawl"];
  apiKey = process.env.KRAWL_API_KEY!;
  k = 10;

  async _getRelevantDocuments(query: string): Promise<Document[]> {
    const r = await fetch("https://api.krawl.sh/search", {
      method: "POST",
      headers: { "X-API-Key": this.apiKey, "Content-Type": "application/json" },
      body: JSON.stringify({ query, limit: this.k }),
    });
    const data = await r.json();
    return (data.results ?? []).map(
      (hit: any) =>
        new Document({
          pageContent: hit.content ?? hit.snippet ?? "",
          metadata: { url: hit.url, title: hit.title, source_type: hit.source_type, score: hit.score },
        }),
    );
  }
}