From bb0b40aae0fcbfdf312494c2bfb1fc3a77e8681c Mon Sep 17 00:00:00 2001 From: Joseph Mutua Date: Wed, 16 Sep 2026 21:42:54 +0300 Subject: [PATCH] feat: add asyncio client with HTTPX --- HISTORY.md | 12 + MANIFEST.in | 1 + README.md | 41 ++- benchmarks/README.md | 35 +++ benchmarks/http_clients.py | 128 +++++++++ docs/index.md | 7 +- docs/user_guide/asyncio.md | 108 ++++++++ docs/user_guide/client-usage.md | 6 +- docs/user_guide/errors-and-timeouts.md | 2 +- docs/user_guide/getting-started.md | 4 +- docs/user_guide/index.md | 1 + docs/user_guide/request-options.md | 86 +++--- pyproject.toml | 6 +- serpapi/core.py | 367 ++++++++++++++++--------- serpapi/exceptions.py | 60 ++-- serpapi/http.py | 262 +++++++++++++++--- serpapi/models.py | 32 ++- tests/docs_example_support.py | 68 +++-- tests/test_async_client.py | 265 ++++++++++++++++++ tests/test_docs_example_runner.py | 90 ++++-- tests/test_exceptions.py | 28 +- tests/test_httpx_transport.py | 121 ++++++++ tests/test_image_upload.py | 19 +- tests/test_output_formats.py | 14 +- tests/test_pagination.py | 13 +- tests/test_timeout.py | 35 ++- 26 files changed, 1460 insertions(+), 351 deletions(-) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/http_clients.py create mode 100644 docs/user_guide/asyncio.md create mode 100644 tests/test_async_client.py create mode 100644 tests/test_httpx_transport.py diff --git a/HISTORY.md b/HISTORY.md index 6496686..b3b029d 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,18 @@ Release History =============== +Unreleased +---------- + +- Added `serpapi.AsyncClient` with async search, archive, account, locations, + image upload, and pagination support. +- Migrated the synchronous HTTP transport from Requests to HTTPX while keeping + the existing `serpapi.Client` API and request-option compatibility. +- Added explicit sync and async client lifecycle management, deterministic + concurrency tests, asyncio documentation, and a local HTTP benchmark. +- Updated the minimum supported Python version to 3.8 to match the SDK's CI + matrix and HTTPX requirements. + 1.1.2 (2026-09-15) ------------------ diff --git a/MANIFEST.in b/MANIFEST.in index 4ddc91b..f28c2a1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ include README.md CONTRIBUTING.md HISTORY.md LICENSE include .readthedocs.yaml include scripts/check_docs_revision.py scripts/publish_docs.py recursive-include tests *.py +recursive-include benchmarks *.py *.md recursive-include docs *.py *.md *.txt *.css Makefile include assets/serpapi-logo.svg assets/serpapi-icon.png prune docs/_build diff --git a/README.md b/README.md index a379667..cd518e1 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,18 @@ Query a vast range of data at scale, including web search results, flight schedu ## Installation -To install the `serpapi` package, simply run the following command: +Install the `serpapi` package with pip or add it to a uv project: ```bash -$ pip install serpapi +pip3 install serpapi ``` +```bash +uv add serpapi +``` + +Python 3.8 or newer is required. + Please note that this package is separate from the legacy `serpapi` module, which is available on PyPi as `google-search-results`. This package is maintained by SerpApi, and is the recommended way to access the SerpApi service from Python. ## Simple Usage @@ -36,6 +42,37 @@ print(results) The `results` variable now contains a `SerpResults` object, which acts just like a standard dictionary, with some convenient functions added on top. +## Async Usage + +Use `AsyncClient` in applications built on `asyncio`. Reuse one client so its +connection pool can serve all concurrent requests: + +```python +import asyncio +import os + +import serpapi + + +async def main(): + async with serpapi.AsyncClient( + api_key=os.environ["SERPAPI_KEY"] + ) as client: + results = await asyncio.gather( + client.search(engine="google", q="coffee"), + client.search(engine="google", q="tea"), + client.search(engine="google", q="pizza"), + ) + print([result["search_metadata"]["id"] for result in results]) + + +asyncio.run(main()) +``` + +The synchronous `Client` and module-level helpers remain available. See the +[Asyncio Client guide](https://serpapi-python.readthedocs.io/en/latest/user_guide/asyncio.html) +for lifecycle, pagination, error-handling, and upload examples. + This example runs a search for "coffee" on Google. It then returns the results as a regular Python Hash. See the [playground](https://serpapi.com/playground) to generate your own code. diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..c9d80a7 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,35 @@ +# HTTP client benchmark + +This benchmark compares the previous `requests.Session` transport, the +synchronous HTTPX-backed `serpapi.Client`, and concurrent +`serpapi.AsyncClient` calls. + +It runs against a local threaded server with a configurable response delay, so +it does not need an API key, consume searches, or mix client behavior with +internet and SerpApi server variance. + +From the repository root, run: + +```bash +uv run --with requests python benchmarks/http_clients.py +``` + +Or install the development package and benchmark-only dependency with pip: + +```bash +pip3 install -e . requests +python benchmarks/http_clients.py +``` + +Each result is the median of three runs and includes the observed range. Client +construction and shutdown are excluded consistently. Change the workload with +`--count`, `--delay`, and `--repeats`. For example: + +```bash +uv run --with requests python benchmarks/http_clients.py --count 50 --delay 0.1 --repeats 5 +``` + +The sequential `requests` and HTTPX results show transport overhead under the +same workload. The async result demonstrates throughput when independent +I/O-bound calls overlap; it does not claim that one SerpApi search becomes +faster. diff --git a/benchmarks/http_clients.py b/benchmarks/http_clients.py new file mode 100644 index 0000000..f88e6c5 --- /dev/null +++ b/benchmarks/http_clients.py @@ -0,0 +1,128 @@ +"""Compare sequential requests with concurrent SerpApi AsyncClient calls. + +The benchmark uses a local delayed HTTP server. It measures client-side +concurrency without consuming SerpApi searches or introducing internet and API +server variance. +""" + +import argparse +import asyncio +import json +import statistics +import threading +import time +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import requests + +import serpapi + + +class DelayedJSONHandler(BaseHTTPRequestHandler): + delay = 0.05 + + def do_GET(self): + time.sleep(self.delay) + payload = json.dumps({"search_metadata": {"status": "Success"}}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, format, *args): + pass + + +@contextmanager +def delayed_server(delay): + DelayedJSONHandler.delay = delay + server = ThreadingHTTPServer(("127.0.0.1", 0), DelayedJSONHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + host, port = server.server_address + yield f"http://{host}:{port}" + finally: + server.shutdown() + server.server_close() + thread.join() + + +def measure_requests(base_url, count): + session = requests.Session() + session.trust_env = False + try: + started = time.perf_counter() + for _ in range(count): + response = session.get(f"{base_url}/search", timeout=10) + response.raise_for_status() + return time.perf_counter() - started + finally: + session.close() + + +def measure_sync_client(base_url, count): + client = serpapi.Client(trust_env=False, timeout=10) + client.BASE_DOMAIN = base_url + try: + started = time.perf_counter() + for index in range(count): + client.search(q=f"query-{index}") + return time.perf_counter() - started + finally: + client.close() + + +async def measure_async_client(base_url, count): + client = serpapi.AsyncClient(trust_env=False, timeout=10) + async with client: + client.BASE_DOMAIN = base_url + started = time.perf_counter() + await asyncio.gather( + *(client.search(q=f"query-{index}") for index in range(count)) + ) + return time.perf_counter() - started + + +def summarize(label, timings): + median = statistics.median(timings) + spread = f"{min(timings):.3f}-{max(timings):.3f}s" + print(f"{label:<29} {median:.3f}s median ({spread})") + return median + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--count", type=int, default=20) + parser.add_argument("--delay", type=float, default=0.05) + parser.add_argument("--repeats", type=int, default=3) + args = parser.parse_args() + if args.count < 1: + parser.error("--count must be at least 1") + if args.delay < 0: + parser.error("--delay must not be negative") + if args.repeats < 1: + parser.error("--repeats must be at least 1") + + with delayed_server(args.delay) as base_url: + requests_timings = [ + measure_requests(base_url, args.count) for _ in range(args.repeats) + ] + sync_timings = [ + measure_sync_client(base_url, args.count) for _ in range(args.repeats) + ] + async_timings = [ + asyncio.run(measure_async_client(base_url, args.count)) + for _ in range(args.repeats) + ] + + summarize("requests.Session sequential:", requests_timings) + sync_median = summarize("serpapi.Client sequential:", sync_timings) + async_median = summarize("AsyncClient concurrent:", async_timings) + print(f"async vs sync speedup: {sync_median / async_median:.2f}x") + + +if __name__ == "__main__": + main() diff --git a/docs/index.md b/docs/index.md index f5c4537..7bdb7ea 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,7 +20,7 @@ Run one of these commands in your terminal. Use `pip` to install into your Pytho :::{tab-item} pip ```bash -pip install serpapi +pip3 install serpapi ``` ::: @@ -35,7 +35,7 @@ uv add serpapi :::: -The package requires Python 3.6 or newer. +The package requires Python 3.8 or newer. ## First Request @@ -92,7 +92,7 @@ Use the same search parameter names as the [SerpApi API documentation](https://s - [Output Formats](user_guide/output-formats.md) explains when to use JSON, Markdown, or HTML. - [Migration Guide](user_guide/migrating-from-google-search-results.md) shows how to replace `google-search-results` with this package. - [Parameters and Engines](user_guide/parameters-and-engines.md) explains search parameters and how to test them in the [SerpApi Playground](https://serpapi.com/playground). -- For scripts that collect results, see [pagination](user_guide/pagination.md), [timeouts and errors](user_guide/errors-and-timeouts.md), [request options](user_guide/request-options.md), and [account and locations](user_guide/account-and-locations.md). +- For concurrent applications and scripts that collect results, see the [Asyncio Client](user_guide/asyncio.md), [pagination](user_guide/pagination.md), [timeouts and errors](user_guide/errors-and-timeouts.md), [request options](user_guide/request-options.md), and [account and locations](user_guide/account-and-locations.md). - For searches you retrieve later, selected response fields, and data retention settings, see [Async Search Archive](user_guide/async-search-archive.md), [JSON Restrictor](user_guide/json-restrictor.md), and [Zero Trace](user_guide/zero-trace.md). - [Examples](examples/index.md) includes searches for web pages, AI answers, local businesses, products, travel, finance, trends, jobs, media, apps, and research. @@ -132,6 +132,7 @@ user_guide/migrating-from-google-search-results :caption: Advanced Usage user_guide/async-search-archive +user_guide/asyncio user_guide/threading user_guide/multiprocessing user_guide/zero-trace diff --git a/docs/user_guide/asyncio.md b/docs/user_guide/asyncio.md new file mode 100644 index 0000000..2c74ad6 --- /dev/null +++ b/docs/user_guide/asyncio.md @@ -0,0 +1,108 @@ +--- +title: "Asyncio Client" +description: "Run concurrent SerpApi requests with AsyncClient and asyncio." +--- + +# Asyncio Client + +`serpapi.AsyncClient` provides non-blocking versions of the SDK methods for +applications that use Python's `asyncio` event loop. This is separate from the +[Async Search Archive](async-search-archive.md), which is a SerpApi API feature +for retrieving a search after the server finishes processing it. + +## Create and Close a Client + +Use an async context manager so the connection pool is always closed: + +```python +import asyncio +import os + +import serpapi + + +async def main(): + async with serpapi.AsyncClient( + api_key=os.environ["SERPAPI_KEY"], + timeout=20, + ) as client: + results = await client.search(engine="google", q="coffee") + print(results["search_metadata"]["id"]) + + +asyncio.run(main()) +``` + +When a context manager does not fit the application lifecycle, call +`await client.aclose()` during shutdown. Create one client per event loop and +reuse it instead of constructing a client for every request. + +## Run Independent Searches Concurrently + +`asyncio.gather()` lets other requests make progress while one request waits +for network I/O: + +```python +async def search_many(client): + return await asyncio.gather( + client.search(engine="google", q="coffee"), + client.search(engine="google", q="tea"), + client.search(engine="google", q="pizza"), + ) +``` + +Concurrency improves throughput for independent I/O-bound requests. It does +not make one search finish faster, and each call still consumes a search from +the account. + +## Other Async Methods + +The async client supports the same endpoint parameters and response formats as +the synchronous client: + +```python +async def inspect_account(client): + account = await client.account() + locations = await client.locations(q="Austin", limit=3) + archived = await client.search_archive(search_id="search-id") + upload = await client.upload_image("image.png") + return account, locations, archived, upload +``` + +The example above is an illustrative fragment and assumes it runs inside the +same async function and client context as the first example. + +## Async Pagination + +JSON search responses are `AsyncSerpResults` objects. Fetch one more page with +`await`, or iterate through several pages with `async for`: + +```python +async def read_pages(results): + next_page = await results.next_page() + + async for page in results.yield_pages(max_pages=5): + for item in page.get("organic_results", []): + print(item.get("title")) + + return next_page +``` + +## Error Handling + +Async methods raise the same SerpApi exceptions as synchronous methods: + +```python +async def safe_search(client): + try: + return await client.search(engine="google", q="coffee") + except serpapi.TimeoutError: + print("The request timed out.") + except serpapi.HTTPConnectionError: + print("Could not connect to SerpApi.") + except serpapi.HTTPError as exc: + print(exc.status_code, exc.error) +``` + +See [Errors and Timeouts](errors-and-timeouts.md) and +[Request Options](request-options.md) for shared configuration details. diff --git a/docs/user_guide/client-usage.md b/docs/user_guide/client-usage.md index 8680c10..e82560f 100644 --- a/docs/user_guide/client-usage.md +++ b/docs/user_guide/client-usage.md @@ -5,7 +5,7 @@ description: "Use the SerpApi client, module helpers, request options, and respo # Client Usage -Create a `serpapi.Client` to reuse your API key, timeout, and HTTP connection settings across requests. +Create a `serpapi.Client` to reuse your API key, timeout, and pooled HTTP connections across requests. For an asyncio application, see [Asyncio Client](asyncio.md). ## Create a Client @@ -86,14 +86,14 @@ locations = client.locations(q="Austin", limit=3) ## Request Options -`search()`, `search_archive()`, `account()`, and `locations()` pass these keyword arguments to the underlying `requests` call: +`search()`, `search_archive()`, `account()`, and `locations()` accept these compatibility options and translate them to HTTPX: | Option | Use | | --- | --- | | `timeout` | Set a different timeout, in seconds, for one request. | | `proxies` | Send the request through a proxy. | | `verify` | Check the server's TLS certificate, or use a custom certificate authority bundle. | -| `stream` | Set the `requests` streaming option. The client still reads the response before returning results. | +| `stream` | Accepted for backward compatibility. SDK methods fully read the response before returning results. | | `cert` | Authenticate the request with a client certificate. | For example, set a shorter timeout for this search: diff --git a/docs/user_guide/errors-and-timeouts.md b/docs/user_guide/errors-and-timeouts.md index 58a2814..03be61a 100644 --- a/docs/user_guide/errors-and-timeouts.md +++ b/docs/user_guide/errors-and-timeouts.md @@ -56,7 +56,7 @@ results = client.search( ) ``` -The timeout applies to connecting and waiting for data. A request can take longer than the timeout overall if data continues to arrive. Without a timeout setting, the client can wait indefinitely. See the [Requests timeout documentation](https://requests.readthedocs.io/en/latest/user/quickstart/#timeouts). +The timeout applies to connecting, reading, writing, and acquiring a pooled connection. Without a timeout setting, the SDK preserves its historical behavior and can wait indefinitely. See the [HTTPX timeout documentation](https://www.python-httpx.org/advanced/timeouts/). ## Missing Search IDs diff --git a/docs/user_guide/getting-started.md b/docs/user_guide/getting-started.md index 99490b1..07c5454 100644 --- a/docs/user_guide/getting-started.md +++ b/docs/user_guide/getting-started.md @@ -16,7 +16,7 @@ Run one of these commands in your terminal. Use `pip` for an existing Python env :::{tab-item} pip ```bash -pip install serpapi +pip3 install serpapi ``` ::: @@ -39,7 +39,7 @@ uv pip install serpapi :::: -The package requires Python 3.6 or newer. +The package requires Python 3.8 or newer. Create or sign in to your SerpApi account and copy your API key from the [dashboard](https://serpapi.com/manage-api-key). An API key identifies your account when you make a request. Replace `secret_api_key` below with your key and run the command in your terminal. On Windows, use PowerShell. diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index 9e1fd5b..dfcc231 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -22,6 +22,7 @@ Start with [Getting Started](getting-started.md) to install the package and run ## Advanced Usage - [Async Search Archive](async-search-archive.md) +- [Asyncio Client](asyncio.md) - [Threading](threading.md) - [Multiprocessing](multiprocessing.md) - [Zero Trace and Data Retention](zero-trace.md) diff --git a/docs/user_guide/request-options.md b/docs/user_guide/request-options.md index 60c70db..be150a4 100644 --- a/docs/user_guide/request-options.md +++ b/docs/user_guide/request-options.md @@ -1,27 +1,34 @@ --- title: "Request Options" -description: "Pass requests library options such as timeout, proxies, verify, stream, and cert." +description: "Configure HTTPX timeouts, proxies, TLS verification, streaming compatibility, and client certificates." --- # Request Options -Search parameters such as `engine`, `q`, `location`, `hl`, `gl`, `no_cache`, and `json_restrictor` tell SerpApi what to search for and return. Request options such as `timeout` and `proxies` control how your Python program connects to SerpApi through the `requests` library. +Search parameters such as `engine`, `q`, `location`, `hl`, `gl`, `no_cache`, +and `json_restrictor` tell SerpApi what to search for and return. Request +options control how the Python client connects to SerpApi through HTTPX. The examples use a client created as shown in [Client Usage](client-usage.md#create-a-client). ## Supported Request Options -The client passes these keyword arguments to `requests.Session.request()`: +The synchronous and asynchronous clients accept these options on `search()`, +`search_archive()`, `account()`, `locations()`, and `upload_image()`: | Option | Use | | --- | --- | | `timeout` | Set a different timeout, in seconds, for one request. | -| `proxies` | Send the request through HTTP or HTTPS proxies. | -| `verify` | Check the server's TLS certificate, or use a custom certificate authority bundle. | -| `stream` | Set the `requests` streaming option. The client still reads the response before returning results. | +| `proxies` | Use a requests-style mapping of protocols to proxy URLs. | +| `verify` | Check the server's TLS certificate, disable verification, or use a custom CA bundle. | +| `stream` | Accepted for backward compatibility; SDK methods still fully read responses. | | `cert` | Authenticate the request with a client certificate. | -These options are supported by `search()`, `search_archive()`, `account()`, and `locations()`. +HTTPX configures proxies and TLS on client instances rather than individual +requests. The SDK keeps the options above compatible by creating a scoped +HTTPX client when a request overrides those settings. Configure them on the +SerpApi client when several requests share the same settings so the connection +pool can be reused. ## Per-Request Timeout @@ -41,34 +48,44 @@ results = client.search( ) ``` -`timeout=5` sets the connection and read timeout to five seconds. It is passed to `requests`. See [Errors and Timeouts](errors-and-timeouts.md#connection-errors-and-timeouts) for timeout behavior. +The same options work with `AsyncClient`. See [Errors and Timeouts](errors-and-timeouts.md#connection-errors-and-timeouts) for timeout behavior. ## Proxies -To connect through a proxy server, pass a dictionary that maps the request protocol to the proxy URL: +For connection reuse, set one proxy when constructing the client: + + +```python +client = serpapi.Client( + api_key="secret_api_key", + proxy="http://proxy.example.com:8080", +) +results = client.search(engine="google", q="coffee") +``` + +Existing requests-style per-call mappings remain supported: ```python results = client.search( engine="google", q="coffee", - proxies={ - "https": "http://proxy.example.com:8080", - }, + proxies={"https": "http://proxy.example.com:8080"}, ) ``` -Keep proxy credentials in environment variables or your secret manager rather than hardcoding them in source files. +Keep proxy credentials in environment variables or a secret manager rather +than hardcoding them in source files. ## TLS Verification -By default, `requests` checks the server's TLS certificate to verify its identity. If your network uses a private certificate authority (CA), pass the path to its trusted certificate bundle: +By default, HTTPX checks the server's TLS certificate. If your network uses a +private certificate authority, configure its trusted bundle on the client: ```python -results = client.search( - engine="google", - q="coffee", +client = serpapi.Client( + api_key="secret_api_key", verify="/path/to/ca-bundle.pem", ) ``` @@ -77,24 +94,20 @@ Only disable verification for controlled local debugging: ```python -results = client.search( - engine="google", - q="coffee", - verify=False, -) +results = client.search(engine="google", q="coffee", verify=False) ``` Do not use `verify=False` in production code. ## Client Certificates -Some servers or proxies require a certificate to identify the client. Pass its file path with `cert`, or pass a `(cert, key)` tuple if the certificate and private key are in separate files: +Some servers or proxies require a certificate to identify the client. Pass its +file path or a `(certificate, key)` pair when constructing the client: ```python -results = client.search( - engine="google", - q="coffee", +client = serpapi.Client( + api_key="secret_api_key", cert=("/path/to/client-cert.pem", "/path/to/client-key.pem"), ) ``` @@ -108,30 +121,19 @@ results = client.search( engine="google", q="coffee", location="Austin, Texas", - hl="en", - gl="us", - json_restrictor="organic_results[].{title, link}", timeout=10, ) ``` -The client sends the arguments to two places: - -- `timeout`, `proxies`, `verify`, `stream`, and `cert` are passed to `requests`. -- All remaining keyword arguments are sent to SerpApi as API parameters. +The SDK sends `timeout`, `proxies`, `verify`, `stream`, and `cert` to its HTTP +compatibility layer. All remaining keyword arguments are SerpApi parameters. -If you pass a parameter dictionary and keyword arguments together, the search parameters in the keyword arguments update your dictionary: +If you pass a parameter dictionary and keyword arguments together, the keyword +parameters update that dictionary, preserving the existing SDK behavior: ```python params = {"engine": "google", "q": "coffee"} results = client.search(params, location="Austin, Texas", timeout=10) ``` -In this example, `location` is added to the SerpApi request parameters, while `timeout` is passed to `requests`. - -When you want to keep the original dictionary unchanged, pass a copy: - -```python -params = {"engine": "google", "q": "coffee"} -results = client.search(params.copy(), location="Austin, Texas") -``` +Pass `params.copy()` when the original dictionary must remain unchanged. diff --git a/pyproject.toml b/pyproject.toml index 2c259a9..8f20bfb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,8 +9,8 @@ description = "The official Python client for SerpApi.com." readme = "README.md" license = { text = "MIT" } authors = [{ name = "SerpApi", email = "support@serpapi.com" }] -requires-python = ">=3.6" -dependencies = ["requests"] +requires-python = ">=3.8" +dependencies = ["httpx>=0.28,<1"] keywords = [ "scrape", "serp", "api", "serpapi", "scraping", "json", "search", "localized", "rank", "google", "bing", "baidu", "yandex", "yahoo", @@ -20,8 +20,6 @@ keywords = [ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/serpapi/core.py b/serpapi/core.py index f24e089..682085c 100644 --- a/serpapi/core.py +++ b/serpapi/core.py @@ -1,194 +1,295 @@ +import atexit import io import os +from contextlib import contextmanager -from .http import HTTPClient from .exceptions import SearchIDNotProvided -from .models import SerpResults +from .http import REQUEST_OPTIONS, AsyncHTTPClient, HTTPClient +from .models import AsyncSerpResults, SerpResults + +__all__ = [ + "AsyncClient", + "AsyncSerpResults", + "Client", + "SerpResults", + "account", + "locations", + "search", + "search_archive", + "upload_image", +] + +DASHBOARD_URL = "https://serpapi.com/dashboard" + + +def _split_parameters(params, kwargs): + if params is None: + params = {} + + request_kwargs = {} + for key in REQUEST_OPTIONS: + if key in kwargs: + request_kwargs[key] = kwargs.pop(key) + + if kwargs: + params.update(kwargs) + return params, request_kwargs + + +def _search_id(params): + try: + return params["search_id"] + except KeyError: + raise SearchIDNotProvided( + f"Please provide 'search_id', found here: {DASHBOARD_URL}" + ) from None + + +@contextmanager +def _image_stream(image): + if isinstance(image, (str, os.PathLike)): + with open(image, "rb") as image_file: + yield image_file + return + if isinstance(image, io.TextIOBase): + raise TypeError( + "image file must be opened in binary mode, e.g. open(path, 'rb')" + ) + yield image class Client(HTTPClient): - """A class that handles API requests to SerpApi in a user-friendly manner. - - Store your API key in an environment variable, then create a client with - ``serpapi.Client(api_key=os.environ["SERPAPI_KEY"])``. - - :param api_key: The API Key to use for SerpApi.com. - :param timeout: The default timeout to use for requests. + """A synchronous client for SerpApi. + + :param api_key: The API key to use for SerpApi.com. + :param timeout: Default request timeout in seconds. ``None`` waits without + a timeout, matching previous SDK releases. + :param proxy: Optional HTTPX proxy URL used by every request. + :param verify: TLS verification setting or custom CA bundle. + :param cert: Client certificate path or ``(certificate, key)`` pair. + :param trust_env: Whether HTTPX reads proxy and certificate environment + variables. """ - DASHBOARD_URL = "https://serpapi.com/dashboard" - - def __init__(self, *, api_key=None, timeout=None): - super().__init__(api_key=api_key, timeout=timeout) + DASHBOARD_URL = DASHBOARD_URL + + def __init__( + self, + *, + api_key=None, + timeout=None, + proxy=None, + verify=True, + cert=None, + trust_env=True, + ): + super().__init__( + api_key=api_key, + timeout=timeout, + proxy=proxy, + verify=verify, + cert=cert, + trust_env=trust_env, + ) def __repr__(self): return "" - def search(self, params: dict = None, **kwargs): - """Fetch a page of results from SerpApi. + def search(self, params=None, **kwargs): + """Fetch one page of search results from SerpApi. - Returns a ``serpapi.SerpResults`` object for JSON responses, or text - when ``output="html"`` or ``output="md"`` is requested. Prefer passing SerpApi engine - parameters as keyword arguments. A parameter dictionary is also accepted - when your code already has parameters in a mapping. + :param params: Optional mapping of SerpApi search parameters. + :param kwargs: Additional search parameters or supported request + options. + :return: ``SerpResults`` for JSON, or ``str`` for HTML and Markdown. + """ + params, request_kwargs = _split_parameters(params, kwargs) + response = self.request("GET", "/search", params=params, **request_kwargs) + return SerpResults.from_http_response(response, client=self) + def search_archive(self, params=None, **kwargs): + """Get a result from the SerpApi Search Archive API. - :param params: Optional mapping of SerpApi search parameters such as ``engine``, ``q``, ``location``, and ``output``. - :param kwargs: Additional SerpApi parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. + :param params: Archive parameters including ``search_id``. + :param kwargs: Additional archive parameters or request options. + :return: ``SerpResults`` for JSON, or ``str`` for HTML and Markdown. + :raises SearchIDNotProvided: if no ``search_id`` is supplied. + """ + params, request_kwargs = _split_parameters(params, kwargs) + search_id = _search_id(params) + response = self.request( + "GET", f"/searches/{search_id}", params=params, **request_kwargs + ) + return SerpResults.from_http_response(response, client=self) + def upload_image(self, image, **kwargs): + """Upload an image to SerpApi's Image API. - **Learn more**: https://serpapi.com/search-api + :param image: Path or open binary file containing a supported image. + :param kwargs: Multipart fields or supported request options. + :return: Parsed JSON response containing the temporary ``image_id``. """ - if params is None: - params = {} + _, request_kwargs = _split_parameters({}, kwargs) + data = kwargs + if "api_key" not in data: + data["api_key"] = self.api_key - # These are arguments that should be passed to the underlying requests.request call. - request_kwargs = {} - for key in ["timeout", "proxies", "verify", "stream", "cert"]: - if key in kwargs: - request_kwargs[key] = kwargs.pop(key) + with _image_stream(image) as image_stream: + response = self.request( + "POST", + "/image", + params={}, + data=data, + files={"image": image_stream}, + **request_kwargs, + ) + return response.json() - if kwargs: - params.update(kwargs) + def locations(self, params=None, **kwargs): + """Get a list of supported Google locations. - r = self.request("GET", "/search", params=params, **request_kwargs) + :param params: Location API parameters such as ``q`` and ``limit``. + :param kwargs: Additional location parameters or request options. + :return: A list of matching location dictionaries. + """ + params, request_kwargs = _split_parameters(params, kwargs) + response = self.request( + "GET", + "/locations.json", + params=params, + assert_200=True, + **request_kwargs, + ) + return response.json() - return SerpResults.from_http_response(r, client=self) + def account(self, params=None, **kwargs): + """Get SerpApi account information. - def search_archive(self, params: dict = None, **kwargs): - """Get a result from the SerpApi Search Archive API. + :param params: Optional Account API parameters. + :param kwargs: Additional account parameters or request options. + :return: The account response as a dictionary. + """ + params, request_kwargs = _split_parameters(params, kwargs) + response = self.request( + "GET", + "/account.json", + params=params, + assert_200=True, + **request_kwargs, + ) + return response.json() - :param params: Archive parameters. Must include ``search_id``. ``output`` accepts ``json`` (default), ``html``, or ``md``. - :param kwargs: Additional archive parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. - **Learn more**: https://serpapi.com/search-archive-api - """ - if params is None: - params = {} - - # These are arguments that should be passed to the underlying requests.request call. - request_kwargs = {} - for key in ["timeout", "proxies", "verify", "stream", "cert"]: - if key in kwargs: - request_kwargs[key] = kwargs.pop(key) - - if kwargs: - params.update(kwargs) - - try: - search_id = params["search_id"] - except KeyError: - raise SearchIDNotProvided( - f"Please provide 'search_id', found here: { self.DASHBOARD_URL }" - ) +class AsyncClient(AsyncHTTPClient): + """An asynchronous client for SerpApi. - r = self.request("GET", f"/searches/{ search_id }", params=params, **request_kwargs) - return SerpResults.from_http_response(r, client=self) + Reuse one instance for concurrent requests and close it with ``aclose()``, + or use ``async with serpapi.AsyncClient(...) as client``. - def upload_image(self, image, **kwargs): - """Upload an image to SerpApi's Image API. + Constructor arguments match :class:`Client`. + """ - ``image`` can be a filesystem path or an open binary file object. The - returned dictionary contains an ``image_id`` that can be passed to - :meth:`search` for engines that accept uploaded images, such as Google - Lens. + DASHBOARD_URL = DASHBOARD_URL + + def __init__( + self, + *, + api_key=None, + timeout=None, + proxy=None, + verify=True, + cert=None, + trust_env=True, + ): + super().__init__( + api_key=api_key, + timeout=timeout, + proxy=proxy, + verify=verify, + cert=cert, + trust_env=trust_env, + ) - :param image: a path or open binary file object containing a JPG/JPEG, - PNG, or WebP image no larger than 500 KB. - :param api_key: the API Key to use for SerpApi.com. - :param **: any additional multipart form fields to pass to the API. + def __repr__(self): + return "" - **Learn more**: https://serpapi.com/image-api + async def search(self, params=None, **kwargs): + """Asynchronously fetch one page of search results from SerpApi. + + Parameters and return values match :meth:`Client.search`. + """ + params, request_kwargs = _split_parameters(params, kwargs) + response = await self.request("GET", "/search", params=params, **request_kwargs) + return AsyncSerpResults.from_http_response(response, client=self) + + async def search_archive(self, params=None, **kwargs): + """Asynchronously get a result from the Search Archive API. + + Parameters, return values, and errors match + :meth:`Client.search_archive`. """ - request_kwargs = {} - for key in ["timeout", "proxies", "verify", "stream", "cert"]: - if key in kwargs: - request_kwargs[key] = kwargs.pop(key) + params, request_kwargs = _split_parameters(params, kwargs) + search_id = _search_id(params) + response = await self.request( + "GET", f"/searches/{search_id}", params=params, **request_kwargs + ) + return AsyncSerpResults.from_http_response(response, client=self) + + async def upload_image(self, image, **kwargs): + """Asynchronously upload an image to SerpApi's Image API. + Parameters and return values match :meth:`Client.upload_image`. + """ + _, request_kwargs = _split_parameters({}, kwargs) data = kwargs if "api_key" not in data: data["api_key"] = self.api_key - image_file = None - try: - if isinstance(image, (str, os.PathLike)): - image_file = open(image, "rb") - image = image_file - elif isinstance(image, io.TextIOBase): - raise TypeError( - "image file must be opened in binary mode, e.g. open(path, 'rb')" - ) - - r = self.request( + with _image_stream(image) as image_stream: + response = await self.request( "POST", "/image", params={}, data=data, - files={"image": image}, + files={"image": image_stream}, **request_kwargs, ) - return r.json() - finally: - if image_file is not None: - image_file.close() - - def locations(self, params: dict = None, **kwargs): - """Get a list of supported Google locations. - + return response.json() - :param params: Location API parameters such as ``q`` and ``limit``. - :param kwargs: Additional location parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. + async def locations(self, params=None, **kwargs): + """Asynchronously get supported Google locations. - **Learn more**: https://serpapi.com/locations-api + Parameters and return values match :meth:`Client.locations`. """ - if params is None: - params = {} - - # These are arguments that should be passed to the underlying requests.request call. - request_kwargs = {} - for key in ["timeout", "proxies", "verify", "stream", "cert"]: - if key in kwargs: - request_kwargs[key] = kwargs.pop(key) - - if kwargs: - params.update(kwargs) - - r = self.request( + params, request_kwargs = _split_parameters(params, kwargs) + response = await self.request( "GET", "/locations.json", params=params, assert_200=True, **request_kwargs, ) - return r.json() - - def account(self, params: dict = None, **kwargs): - """Get SerpApi account information. + return response.json() - :param params: Account API parameters. - :param kwargs: Additional account parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. + async def account(self, params=None, **kwargs): + """Asynchronously get SerpApi account information. - **Learn more**: https://serpapi.com/account-api + Parameters and return values match :meth:`Client.account`. """ - - if params is None: - params = {} - - # These are arguments that should be passed to the underlying requests.request call. - request_kwargs = {} - for key in ["timeout", "proxies", "verify", "stream", "cert"]: - if key in kwargs: - request_kwargs[key] = kwargs.pop(key) - - if kwargs: - params.update(kwargs) - - r = self.request("GET", "/account.json", params=params, assert_200=True, **request_kwargs) - return r.json() + params, request_kwargs = _split_parameters(params, kwargs) + response = await self.request( + "GET", + "/account.json", + params=params, + assert_200=True, + **request_kwargs, + ) + return response.json() -# An un-authenticated client instance. +# Backward-compatible synchronous module helpers use one shared client. _client = Client() +atexit.register(_client.close) search = _client.search search_archive = _client.search_archive upload_image = _client.upload_image diff --git a/serpapi/exceptions.py b/serpapi/exceptions.py index c268c4d..a873e59 100644 --- a/serpapi/exceptions.py +++ b/serpapi/exceptions.py @@ -1,51 +1,55 @@ -import requests +import httpx class SerpApiError(Exception): """Base class for exceptions in this module.""" - pass - class APIKeyNotProvided(ValueError, SerpApiError): """API key is not provided.""" - pass - class SearchIDNotProvided(ValueError, SerpApiError): """Search ID is not provided.""" - pass - -class HTTPError(requests.exceptions.HTTPError, SerpApiError): - """HTTP Error.""" +class HTTPError(httpx.HTTPError, SerpApiError): + """An unsuccessful HTTP response from SerpApi.""" def __init__(self, original_exception): - if (isinstance(original_exception, requests.exceptions.HTTPError)): - http_error_exception: requests.exceptions.HTTPError = original_exception - - self.status_code = http_error_exception.response.status_code + self.original_exception = original_exception + request = getattr(original_exception, "request", None) + self.response = getattr(original_exception, "response", None) + self.status_code = ( + self.response.status_code if self.response is not None else -1 + ) + self.error = None + + if self.response is not None: try: - self.error = http_error_exception.response.json().get("error", None) - except requests.exceptions.JSONDecodeError: - self.error = None - else: - self.status_code = -1 - self.error = None - - super().__init__(*original_exception.args, response=getattr(original_exception, 'response', None), request=getattr(original_exception, 'request', None)) - + payload = self.response.json() + if isinstance(payload, dict): + self.error = payload.get("error") + except ValueError: + pass + message = str(original_exception) + httpx.HTTPError.__init__(self, message) + if request is not None: + self.request = request -class HTTPConnectionError(HTTPError, requests.exceptions.ConnectionError, SerpApiError): - """Connection Error.""" - pass +class HTTPConnectionError(HTTPError): + """A network error while connecting to or reading from SerpApi.""" -class TimeoutError(requests.exceptions.Timeout, SerpApiError): - """Timeout Error.""" +class TimeoutError(httpx.TimeoutException, SerpApiError): + """A request to SerpApi exceeded its configured timeout.""" - pass + def __init__(self, original_exception): + self.original_exception = original_exception + httpx.TimeoutException.__init__( + self, + str(original_exception), + request=getattr(original_exception, "request", None), + ) diff --git a/serpapi/http.py b/serpapi/http.py index 4ea1a33..e7d3550 100644 --- a/serpapi/http.py +++ b/serpapi/http.py @@ -1,71 +1,247 @@ -import requests +import os +import ssl +from collections.abc import Mapping +from urllib.parse import urlsplit, urlunsplit + +import httpx -from .exceptions import ( - HTTPError, - HTTPConnectionError, - TimeoutError, -) from .__version__ import __version__ +from .exceptions import HTTPConnectionError, HTTPError, TimeoutError + +REQUEST_OPTIONS = ("timeout", "proxies", "verify", "stream", "cert") +_UNSET = object() + + +def _ssl_configuration(verify=True, cert=None): + """Translate requests-style TLS options into an HTTPX configuration.""" + if cert is None and not isinstance(verify, (str, os.PathLike)): + return verify + + if isinstance(verify, ssl.SSLContext): + context = verify + elif verify is False: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + elif isinstance(verify, (str, os.PathLike)): + context = ssl.create_default_context(cafile=os.fspath(verify)) + else: + context = ssl.create_default_context() + + if cert is not None: + if isinstance(cert, (tuple, list)): + context.load_cert_chain(os.fspath(cert[0]), os.fspath(cert[1])) + else: + context.load_cert_chain(os.fspath(cert)) + + return context + +def _proxy_configuration(proxies, *, asynchronous, verify): + if not isinstance(proxies, Mapping): + return {"proxy": proxies} -class HTTPClient: - """This class handles outgoing HTTP requests to SerpApi.com.""" + transport_class = httpx.AsyncHTTPTransport if asynchronous else httpx.HTTPTransport + mounts = {} + for scheme, proxy_url in proxies.items(): + mount = scheme if "://" in scheme else f"{scheme}://" + mounts[mount] = ( + None + if proxy_url is None + else transport_class(proxy=proxy_url, verify=verify) + ) + return {"mounts": mounts} + + +class _HTTPClientBase: + """Shared request preparation for synchronous and asynchronous clients.""" BASE_DOMAIN = "https://serpapi.com" USER_AGENT = f"serpapi-python, v{__version__}" - def __init__(self, *, api_key=None, timeout=None): - # Used to authenticate requests. - # TODO: do we want to support the environment variable? Seems like a security risk. + def __init__( + self, + *, + api_key=None, + timeout=None, + proxy=None, + verify=True, + cert=None, + trust_env=True, + ): self.api_key = api_key self.timeout = timeout - self.session = requests.Session() + self._proxy = proxy + self._verify = verify + self._cert = cert + self._trust_env = trust_env + + def _client_options( + self, *, asynchronous, proxies=_UNSET, verify=_UNSET, cert=_UNSET + ): + selected_verify = self._verify if verify is _UNSET else verify + selected_cert = self._cert if cert is _UNSET else cert + ssl_config = _ssl_configuration(selected_verify, selected_cert) + options = { + "headers": {"User-Agent": self.USER_AGENT}, + "timeout": self.timeout, + "follow_redirects": True, + "trust_env": self._trust_env, + "verify": ssl_config, + } + + selected_proxy = self._proxy if proxies is _UNSET else proxies + if selected_proxy is not None: + options.update( + _proxy_configuration( + selected_proxy, + asynchronous=asynchronous, + verify=ssl_config, + ) + ) + return options - def request(self, method, path, params, *, assert_200=True, **kwargs): - # Inject the API Key into the params. + def _prepare_request(self, path, params, kwargs): request_data = kwargs.get("data") api_key_in_data = isinstance(request_data, dict) and "api_key" in request_data if "api_key" not in params and not api_key_in_data: params["api_key"] = self.api_key - # Build the URL, as needed. - if not path.startswith("http"): - url = self.BASE_DOMAIN + path - else: - url = path + # Match requests: query parameters with a None value are omitted. + params = {key: value for key, value in params.items() if value is not None} + url = path if path.startswith("http") else self.BASE_DOMAIN + path + parsed_url = urlsplit(url) + if parsed_url.query: + params = httpx.QueryParams(parsed_url.query).merge(params) + url = urlunsplit(parsed_url._replace(query="")) + + if self.timeout is not None and "timeout" not in kwargs: + kwargs["timeout"] = self.timeout + + # SDK endpoint methods always consume the response before returning. + # Accept the old option for compatibility, but use HTTPX's safe, + # fully-buffered request behavior. + kwargs.pop("stream", None) + + connection_options = {} + for key in ("proxies", "verify", "cert"): + if key in kwargs: + connection_options[key] = kwargs.pop(key) + + return url, params, kwargs, connection_options - # Make the HTTP request. + +class HTTPClient(_HTTPClientBase): + """Synchronous HTTP transport with connection pooling.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.session = httpx.Client(**self._client_options(asynchronous=False)) + + def request(self, method, path, params, *, assert_200=True, **kwargs): + url, params, kwargs, connection_options = self._prepare_request( + path, params, kwargs + ) try: - headers = {"User-Agent": self.USER_AGENT} + if connection_options: + options = self._client_options( + asynchronous=False, + proxies=connection_options.get("proxies", _UNSET), + verify=connection_options.get("verify", _UNSET), + cert=connection_options.get("cert", _UNSET), + ) + with httpx.Client(**options) as session: + response = session.request( + method=method, + url=url, + params=params, + headers={"User-Agent": self.USER_AGENT}, + **kwargs, + ) + else: + response = self.session.request( + method=method, + url=url, + params=params, + headers={"User-Agent": self.USER_AGENT}, + **kwargs, + ) + except httpx.TimeoutException as exc: + raise TimeoutError(exc) from exc + except httpx.RequestError as exc: + raise HTTPConnectionError(exc) from exc - # Use the default timeout if one was provided to the client. - if self.timeout and "timeout" not in kwargs: - kwargs["timeout"] = self.timeout + if assert_200: + raise_for_status(response) + return response - r = self.session.request( - method=method, url=url, params=params, headers=headers, **kwargs - ) + def close(self): + self.session.close() + + def __enter__(self): + return self - except requests.exceptions.ConnectionError as e: - raise HTTPConnectionError(e) - except requests.exceptions.Timeout as e: - raise TimeoutError(e) + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + +class AsyncHTTPClient(_HTTPClientBase): + """Asynchronous HTTP transport with connection pooling.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.session = httpx.AsyncClient(**self._client_options(asynchronous=True)) + + async def request(self, method, path, params, *, assert_200=True, **kwargs): + url, params, kwargs, connection_options = self._prepare_request( + path, params, kwargs + ) + try: + if connection_options: + options = self._client_options( + asynchronous=True, + proxies=connection_options.get("proxies", _UNSET), + verify=connection_options.get("verify", _UNSET), + cert=connection_options.get("cert", _UNSET), + ) + async with httpx.AsyncClient(**options) as session: + response = await session.request( + method=method, + url=url, + params=params, + headers={"User-Agent": self.USER_AGENT}, + **kwargs, + ) + else: + response = await self.session.request( + method=method, + url=url, + params=params, + headers={"User-Agent": self.USER_AGENT}, + **kwargs, + ) + except httpx.TimeoutException as exc: + raise TimeoutError(exc) from exc + except httpx.RequestError as exc: + raise HTTPConnectionError(exc) from exc - # Raise an exception if the status code is not 200. if assert_200: - try: - raise_for_status(r) - except requests.exceptions.HTTPError as e: - raise HTTPError(e) + raise_for_status(response) + return response + + async def aclose(self): + await self.session.aclose() - return r + async def __aenter__(self): + return self + async def __aexit__(self, exc_type, exc_value, traceback): + await self.aclose() -def raise_for_status(r): - """Raise an exception if the status code is not 200.""" - # TODO: put custom behavior in here for various status codes. +def raise_for_status(response): + """Raise the SerpApi HTTP error type for unsuccessful responses.""" try: - r.raise_for_status() - except requests.exceptions.HTTPError as e: - raise HTTPError(e) + response.raise_for_status() + except httpx.HTTPStatusError as exc: + raise HTTPError(exc) from exc diff --git a/serpapi/models.py b/serpapi/models.py index 3e8a982..96fc6ab 100644 --- a/serpapi/models.py +++ b/serpapi/models.py @@ -1,5 +1,4 @@ import json - from collections import UserDict from .textui import prettify_json @@ -64,7 +63,7 @@ def yield_pages(self, max_pages=1_000): """ current_page_count = 0 - + current_page = self while current_page and current_page_count < max_pages: yield current_page @@ -75,7 +74,6 @@ def yield_pages(self, max_pages=1_000): current_page = current_page.next_page() else: break - @classmethod def from_http_response(cls, r, *, client=None): @@ -93,9 +91,31 @@ def from_http_response(cls, r, *, client=None): return r.text try: - cls = cls(r.json(), client=client) - - return cls + return cls(r.json(), client=client) except ValueError: # If the response is not JSON, return the raw text. return r.text + + +class AsyncSerpResults(SerpResults): + """Search results with asynchronous pagination helpers.""" + + async def next_page(self): + """Asynchronously return the next page of results, if any.""" + if self.next_page_url: + params = {"api_key": self.client.api_key} + response = await self.client.request( + "GET", path=self.next_page_url, params=params + ) + return type(self).from_http_response(response, client=self.client) + + async def yield_pages(self, max_pages=1_000): + """Yield this result and up to ``max_pages - 1`` additional pages.""" + current_page_count = 0 + current_page = self + while current_page and current_page_count < max_pages: + yield current_page + current_page_count += 1 + if current_page_count >= max_pages or not current_page.next_page_url: + break + current_page = await current_page.next_page() diff --git a/tests/docs_example_support.py b/tests/docs_example_support.py index 7d61d72..afaabe3 100644 --- a/tests/docs_example_support.py +++ b/tests/docs_example_support.py @@ -1,17 +1,16 @@ import json import os -from pathlib import Path import re import shutil import signal import subprocess import sys import threading +from pathlib import Path from urllib.parse import parse_qs, urlsplit import serpapi -from serpapi.http import HTTPClient - +from serpapi.http import AsyncHTTPClient, HTTPClient ROOT = Path(__file__).resolve().parents[1] PYTHON_BLOCK_RE = re.compile( @@ -28,7 +27,9 @@ def python_blocks(path): def docs_pages(): paths = [ROOT / "README.md", *sorted((ROOT / "docs").rglob("*.md"))] - return [path for path in paths if "_build" not in path.parts and python_blocks(path)] + return [ + path for path in paths if "_build" not in path.parts and python_blocks(path) + ] def redact(text, api_key=None): @@ -60,13 +61,16 @@ def validate_response(response, path, params): output = params.get("output", "json") if output in ("md", "html"): if not isinstance(result, str) or not result.strip(): - raise AssertionError(f"output={output} did not return a nonempty string") + raise AssertionError( + f"output={output} did not return a nonempty string" + ) elif not isinstance(result, serpapi.SerpResults): raise AssertionError("JSON search did not return SerpResults") def install_http_checks(): original_request = HTTPClient.request + original_async_request = AsyncHTTPClient.request report_path = Path(os.environ["DOCS_EXAMPLE_REPORT_DIR"]) / f"{os.getpid()}.jsonl" lock = threading.Lock() @@ -89,20 +93,46 @@ def checked_request(self, method, path, params, **kwargs): with lock, report_path.open("a") as report: report.write(json.dumps(record) + "\n") + async def checked_async_request(self, method, path, params, **kwargs): + url = urlsplit(path) + query = {key: values[-1] for key, values in parse_qs(url.query).items()} + query.update(params) + record = {"path": url.path, "engine": query.get("engine"), "ok": False} + if not self.timeout and not kwargs.get("timeout"): + kwargs["timeout"] = 30 + try: + response = await original_async_request( + self, method, path, params, **kwargs + ) + validate_response(response, url.path, query) + record["ok"] = True + return response + except Exception as exc: + record["error"] = redact(f"{type(exc).__name__}: {exc}") + raise + finally: + with lock, report_path.open("a") as report: + report.write(json.dumps(record) + "\n") + HTTPClient.request = checked_request + AsyncHTTPClient.request = checked_async_request def script_for_page(path): parts = [ - "import os\nimport serpapi\n" - "from tests.docs_example_support import install_http_checks\n" - "install_http_checks()\n" - 'client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=30)\n' + ( + "import os\nimport serpapi\n" + "from tests.docs_example_support import install_http_checks\n" + "install_http_checks()\n" + 'client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=30)\n' + ) ] for number, block in enumerate(python_blocks(path), start=1): if block["skip"]: if not block["reason"].strip(): - raise RuntimeError(f"Python block {number} needs a docs-test skip reason") + raise RuntimeError( + f"Python block {number} needs a docs-test skip reason" + ) continue code = block["code"] for placeholder in ('"secret_api_key"', "'secret_api_key'"): @@ -119,12 +149,14 @@ def run_page(path, workdir, api_key, timeout=PAGE_TIMEOUT): script.write_text(script_for_page(path)) shutil.copyfile(ROOT / "assets" / "serpapi-icon.png", workdir / "image.png") env = os.environ.copy() - env.update({ - "SERPAPI_KEY": api_key, - "API_KEY": api_key, - "DOCS_EXAMPLE_REPORT_DIR": str(report_dir), - "PYTHONPATH": os.pathsep.join([str(ROOT / "tests"), str(ROOT)]), - }) + env.update( + { + "SERPAPI_KEY": api_key, + "API_KEY": api_key, + "DOCS_EXAMPLE_REPORT_DIR": str(report_dir), + "PYTHONPATH": os.pathsep.join([str(ROOT / "tests"), str(ROOT)]), + } + ) with subprocess.Popen( [sys.executable, str(script)], cwd=workdir, @@ -142,7 +174,9 @@ def run_page(path, workdir, api_key, timeout=PAGE_TIMEOUT): else: process.kill() process.communicate() - raise RuntimeError(f"Example exceeded the {timeout}-second page limit") from None + raise RuntimeError( + f"Example exceeded the {timeout}-second page limit" + ) from None records = [ json.loads(line) diff --git a/tests/test_async_client.py b/tests/test_async_client.py new file mode 100644 index 0000000..6528b45 --- /dev/null +++ b/tests/test_async_client.py @@ -0,0 +1,265 @@ +import asyncio +from io import BytesIO +from urllib.parse import parse_qs + +import httpx +import pytest + +import serpapi + + +def run(coroutine): + return asyncio.run(coroutine) + + +async def replace_session(client, handler): + await client.session.aclose() + client.session = httpx.AsyncClient( + transport=httpx.MockTransport(handler), + follow_redirects=True, + timeout=None, + ) + + +def test_module_exposes_async_client(): + assert serpapi.AsyncClient + + async def exercise(): + client = serpapi.AsyncClient() + try: + assert repr(client) == "" + finally: + await client.aclose() + + run(exercise()) + + +def test_async_client_endpoints_and_context_manager(): + requests = [] + + async def handler(request): + requests.append(request) + path = request.url.path + if path == "/search": + return httpx.Response( + 200, + headers={"Content-Type": "application/json"}, + json={"search_metadata": {"id": "search-123"}}, + ) + if path == "/searches/search-123": + return httpx.Response(200, json={"search_metadata": {"id": "search-123"}}) + if path == "/account.json": + return httpx.Response(200, json={"account_id": "account-123"}) + if path == "/locations.json": + return httpx.Response(200, json=[{"name": "Austin"}]) + if path == "/image": + return httpx.Response(200, json={"image_id": "image-123"}) + raise AssertionError(f"Unexpected path: {path}") + + async def exercise(): + client = serpapi.AsyncClient(api_key="test-key", timeout=12) + await replace_session(client, handler) + session = client.session + async with client: + search = await client.search(engine="google", q="coffee") + archive = await client.search_archive(search_id="search-123") + account = await client.account() + locations = await client.locations(q="Austin", limit=1) + image = BytesIO(b"fake-image") + upload = await client.upload_image(image, zero_trace="true") + assert not image.closed + return search, archive, account, locations, upload, session + + search, archive, account, locations, upload, session = run(exercise()) + + assert isinstance(search, serpapi.AsyncSerpResults) + assert search["search_metadata"]["id"] == "search-123" + assert archive["search_metadata"]["id"] == "search-123" + assert account == {"account_id": "account-123"} + assert locations == [{"name": "Austin"}] + assert upload == {"image_id": "image-123"} + assert session.is_closed + assert len(requests) == 5 + + search_query = parse_qs(requests[0].url.query.decode()) + assert search_query == { + "engine": ["google"], + "q": ["coffee"], + "api_key": ["test-key"], + } + assert requests[0].headers["User-Agent"].startswith("serpapi-python") + assert b"fake-image" in requests[-1].content + assert b"zero_trace" in requests[-1].content + + +def test_async_search_supports_text_output(): + async def handler(request): + return httpx.Response( + 200, + headers={"Content-Type": "text/markdown; charset=utf-8"}, + text="# Coffee", + ) + + async def exercise(): + client = serpapi.AsyncClient(api_key="test-key") + await replace_session(client, handler) + async with client: + return await client.search(engine="google", q="coffee", output="md") + + assert run(exercise()) == "# Coffee" + + +def test_async_pagination_fetches_only_requested_pages(): + requested_pages = [] + + async def handler(request): + start = int(request.url.params.get("start", 0)) + requested_pages.append(start) + page = start // 10 + 1 + data = {"search_information": {"page_number": page}} + if page < 3: + data["serpapi_pagination"] = { + "next": f"https://serpapi.com/search?start={start + 10}" + } + return httpx.Response( + 200, + headers={"Content-Type": "application/json"}, + json=data, + ) + + async def exercise(): + client = serpapi.AsyncClient(api_key="test-key") + await replace_session(client, handler) + async with client: + first = await client.search(engine="google", q="coffee") + pages = [page async for page in first.yield_pages(max_pages=2)] + return pages + + pages = run(exercise()) + assert [page["search_information"]["page_number"] for page in pages] == [1, 2] + assert all(isinstance(page, serpapi.AsyncSerpResults) for page in pages) + assert requested_pages == [0, 10] + + +def test_async_client_runs_requests_concurrently_on_one_session(): + in_flight = 0 + max_in_flight = 0 + + async def handler(request): + nonlocal in_flight, max_in_flight + in_flight += 1 + max_in_flight = max(max_in_flight, in_flight) + await asyncio.sleep(0.02) + in_flight -= 1 + return httpx.Response(200, json={"query": request.url.params["q"]}) + + async def exercise(): + client = serpapi.AsyncClient(api_key="test-key") + await replace_session(client, handler) + session = client.session + async with client: + results = await asyncio.gather( + client.search(q="coffee"), + client.search(q="tea"), + client.search(q="pizza"), + ) + assert client.session is session + return results + + results = run(exercise()) + assert [result["query"] for result in results] == ["coffee", "tea", "pizza"] + assert max_in_flight == 3 + + +@pytest.mark.parametrize( + ("transport_error", "expected_error"), + [ + (httpx.ConnectError("connection failed"), serpapi.HTTPConnectionError), + (httpx.ReadTimeout("request timed out"), serpapi.TimeoutError), + ], +) +def test_async_client_translates_transport_errors(transport_error, expected_error): + async def handler(request): + transport_error.request = request + raise transport_error + + async def exercise(): + client = serpapi.AsyncClient(api_key="test-key") + await replace_session(client, handler) + async with client: + with pytest.raises(expected_error) as failure: + await client.search(q="coffee") + assert failure.value.__cause__ is transport_error + + run(exercise()) + + +def test_async_client_preserves_http_error_details(): + async def handler(request): + return httpx.Response(401, json={"error": "Invalid API key"}) + + async def exercise(): + client = serpapi.AsyncClient(api_key="bad-key") + await replace_session(client, handler) + async with client: + with pytest.raises(serpapi.HTTPError) as failure: + await client.account() + assert failure.value.status_code == 401 + assert failure.value.error == "Invalid API key" + assert failure.value.request.url.path == "/account.json" + assert failure.value.response.status_code == 401 + + run(exercise()) + + +def test_async_archive_requires_search_id_before_request(): + async def exercise(): + client = serpapi.AsyncClient(api_key="test-key") + try: + with pytest.raises(serpapi.SearchIDNotProvided): + await client.search_archive() + finally: + await client.aclose() + + run(exercise()) + + +def test_async_requests_style_options_use_scoped_client(monkeypatch): + created = [] + + class ScopedAsyncClient: + def __init__(self, **options): + self.options = options + created.append(self) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_value, traceback): + for transport in self.options.get("mounts", {}).values(): + if transport is not None: + await transport.aclose() + + async def request(self, **kwargs): + self.request_kwargs = kwargs + request = httpx.Request(kwargs["method"], kwargs["url"]) + return httpx.Response(200, json={"ok": True}, request=request) + + async def exercise(): + client = serpapi.AsyncClient(api_key="test-key") + monkeypatch.setattr(httpx, "AsyncClient", ScopedAsyncClient) + try: + return await client.search( + q="coffee", + proxies={"https": "http://proxy.example.com:8080"}, + verify=False, + stream=True, + ) + finally: + await client.aclose() + + assert run(exercise()) == {"ok": True} + assert len(created) == 1 + assert created[0].options["verify"] is False + assert "https://" in created[0].options["mounts"] + assert "stream" not in created[0].request_kwargs diff --git a/tests/test_docs_example_runner.py b/tests/test_docs_example_runner.py index 507cddd..2518839 100644 --- a/tests/test_docs_example_runner.py +++ b/tests/test_docs_example_runner.py @@ -3,24 +3,29 @@ import sys from textwrap import dedent +import httpx import pytest -import requests from tests.docs_example_support import ROOT, run_page, validate_response - MOCK_REQUEST = """ import json -import requests +import httpx def fake_request(self, **kwargs): - response = requests.Response() - response.status_code = 200 - response.headers['Content-Type'] = 'application/json' - response._content = json.dumps({'organic_results': [{'title': 'Coffee'}]}).encode() - return response + request = httpx.Request(kwargs['method'], kwargs['url']) + return httpx.Response( + 200, + headers={'Content-Type': 'application/json'}, + json={'organic_results': [{'title': 'Coffee'}]}, + request=request, + ) -requests.Session.request = fake_request +async def fake_async_request(self, **kwargs): + return fake_request(self, **kwargs) + +httpx.Client.request = fake_request +httpx.AsyncClient.request = fake_async_request """ @@ -31,7 +36,9 @@ def markdown_page(tmp_path, code): def test_doc_runner_executes_main_guard_and_spawned_workers(tmp_path): - code = MOCK_REQUEST + """ + code = ( + MOCK_REQUEST + + """ from concurrent.futures import ProcessPoolExecutor import multiprocessing @@ -43,6 +50,7 @@ def search_worker(query): with ProcessPoolExecutor(max_workers=2, mp_context=multiprocessing.get_context('spawn')) as pool: assert list(pool.map(search_worker, ['one', 'two'])) == ['Coffee', 'Coffee'] """ + ) page = markdown_page(tmp_path, code) records = run_page(page, tmp_path / "run", "test-key") assert len(records) == 2 @@ -70,16 +78,38 @@ def test_doc_runner_preserves_block_state_and_skips_marked_examples(tmp_path): assert "test-key" not in (tmp_path / "run" / "example.py").read_text() +def test_doc_runner_records_async_client_requests(tmp_path): + page = markdown_page( + tmp_path, + MOCK_REQUEST + + """ +import asyncio + +async def main(): + async with serpapi.AsyncClient(api_key='secret_api_key') as async_client: + results = await async_client.search(q='coffee') + assert results['organic_results'][0]['title'] == 'Coffee' + +asyncio.run(main()) +""", + ) + records = run_page(page, tmp_path / "run", "test-key") + assert records == [{"path": "/search", "engine": None, "ok": True}] + + def test_doc_runner_fails_on_caught_api_errors_and_redacts_key(tmp_path): - code = MOCK_REQUEST.replace( - "{'organic_results': [{'title': 'Coffee'}]}", - "{'error': os.environ['SERPAPI_KEY']}", - ) + """ + code = ( + MOCK_REQUEST.replace( + "{'organic_results': [{'title': 'Coffee'}]}", + "{'error': os.environ['SERPAPI_KEY']}", + ) + + """ try: client.search(q='coffee') except AssertionError: pass """ + ) page = markdown_page(tmp_path, code) with pytest.raises(RuntimeError) as failure: run_page(page, tmp_path / "run", "private-test-key") @@ -90,7 +120,9 @@ def test_doc_runner_fails_on_caught_api_errors_and_redacts_key(tmp_path): def test_doc_runner_rejects_examples_without_requests(tmp_path): - page = markdown_page(tmp_path, "if __name__ == 'docs_examples':\n client.search(q='coffee')") + page = markdown_page( + tmp_path, "if __name__ == 'docs_examples':\n client.search(q='coffee')" + ) with pytest.raises(RuntimeError, match="without making a SerpApi request"): run_page(page, tmp_path / "run", "test-key") @@ -103,10 +135,13 @@ def test_doc_runner_stops_a_stalled_example(tmp_path): @pytest.mark.parametrize("output", ["md", "html"]) def test_doc_response_check_rejects_json_for_text_output(output): - response = requests.Response() - response.status_code = 200 - response.headers["Content-Type"] = "application/json" - response._content = b'{"organic_results": [{"title": "Coffee"}]}' + request = httpx.Request("GET", "https://serpapi.com/search") + response = httpx.Response( + 200, + headers={"Content-Type": "application/json"}, + json={"organic_results": [{"title": "Coffee"}]}, + request=request, + ) with pytest.raises(AssertionError, match="nonempty string"): validate_response(response, "/search", {"output": output}) @@ -116,9 +151,20 @@ def test_doc_gate_fails_without_a_key(): env.pop("SERPAPI_KEY", None) env.pop("API_KEY", None) result = subprocess.run( - [sys.executable, "-m", "pytest", "tests/test_docs_examples.py", - "--require-docs-key", "--collect-only", "-q"], - cwd=ROOT, env=env, capture_output=True, text=True, + [ + sys.executable, + "-m", + "pytest", + "tests/test_docs_examples.py", + "--require-docs-key", + "--collect-only", + "-q", + ], + cwd=ROOT, + env=env, + capture_output=True, + text=True, + check=False, ) assert result.returncode != 0 assert "Live docs checks require SERPAPI_KEY or API_KEY" in result.stderr diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 8ff68f8..332f219 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -1,17 +1,25 @@ -from unittest.mock import Mock -import requests +import httpx + import serpapi def test_http_error(): """Ensure that an HTTPError has the correct status code and error.""" - mock_response = Mock() - mock_response.status_code = 401 - mock_response.json.return_value = { "error": "Invalid API key" } - - requests_error = requests.exceptions.HTTPError(response=mock_response, request=Mock()) - http_error = serpapi.HTTPError(requests_error) - + request = httpx.Request("GET", "https://serpapi.com/account.json") + response = httpx.Response( + 401, + json={"error": "Invalid API key"}, + request=request, + ) + original = httpx.HTTPStatusError( + "401 Unauthorized", + request=request, + response=response, + ) + http_error = serpapi.HTTPError(original) + assert http_error.status_code == 401 assert http_error.error == "Invalid API key" - assert http_error.response == mock_response + assert http_error.response == response + assert http_error.request == request + assert isinstance(http_error, httpx.HTTPError) diff --git a/tests/test_httpx_transport.py b/tests/test_httpx_transport.py new file mode 100644 index 0000000..abee62c --- /dev/null +++ b/tests/test_httpx_transport.py @@ -0,0 +1,121 @@ +from unittest.mock import Mock + +import httpx + +import serpapi + + +def json_response(url="https://serpapi.com/search"): + request = httpx.Request("GET", url) + return httpx.Response(200, json={"ok": True}, request=request) + + +def test_sync_client_context_manager_closes_session(): + with serpapi.Client(api_key="test-key") as client: + session = client.session + assert not session.is_closed + assert session.is_closed + + +def test_client_default_preserves_no_timeout_and_redirect_behavior(): + client = serpapi.Client(api_key="test-key") + try: + assert client.session.timeout.connect is None + assert client.session.timeout.read is None + assert client.session.follow_redirects is True + finally: + client.close() + + +def test_pagination_url_keeps_repeated_query_parameters(): + captured = [] + + def handler(request): + captured.append(request) + return httpx.Response(200, json={"ok": True}) + + client = serpapi.Client(api_key="test-key") + client.session.close() + client.session = httpx.Client(transport=httpx.MockTransport(handler)) + try: + client.request( + "GET", + "https://serpapi.com/search?filter=a&filter=b&page=2", + params={}, + ) + finally: + client.close() + + assert captured[0].url.params.multi_items() == [ + ("filter", "a"), + ("filter", "b"), + ("page", "2"), + ("api_key", "test-key"), + ] + + +def test_requests_style_per_call_options_use_scoped_httpx_client(monkeypatch): + created = [] + + class ScopedClient: + def __init__(self, **options): + self.options = options + created.append(self) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + for transport in self.options.get("mounts", {}).values(): + if transport is not None: + transport.close() + + def request(self, **kwargs): + self.request_kwargs = kwargs + return json_response(kwargs["url"]) + + client = serpapi.Client(api_key="test-key") + client.session.request = Mock(side_effect=AssertionError("pooled client used")) + monkeypatch.setattr(httpx, "Client", ScopedClient) + try: + result = client.search( + q="coffee", + proxies={"https": "http://proxy.example.com:8080"}, + verify=False, + stream=True, + ) + finally: + client.close() + + assert result == {"ok": True} + assert len(created) == 1 + assert created[0].options["verify"] is False + assert "https://" in created[0].options["mounts"] + assert "stream" not in created[0].request_kwargs + + +def test_sync_client_translates_connection_and_timeout_errors(): + request = httpx.Request("GET", "https://serpapi.com/search") + client = serpapi.Client(api_key="test-key") + try: + client.session.request = Mock( + side_effect=httpx.ConnectError("failed", request=request) + ) + try: + client.search(q="coffee") + except serpapi.HTTPConnectionError as error: + assert isinstance(error.__cause__, httpx.ConnectError) + else: + raise AssertionError("Expected HTTPConnectionError") + + client.session.request = Mock( + side_effect=httpx.ReadTimeout("slow", request=request) + ) + try: + client.search(q="coffee") + except serpapi.TimeoutError as error: + assert isinstance(error.__cause__, httpx.ReadTimeout) + else: + raise AssertionError("Expected TimeoutError") + finally: + client.close() diff --git a/tests/test_image_upload.py b/tests/test_image_upload.py index 3ed5a96..ffd64cf 100644 --- a/tests/test_image_upload.py +++ b/tests/test_image_upload.py @@ -1,17 +1,15 @@ from io import BytesIO, StringIO from unittest.mock import Mock +import httpx import pytest -import requests import serpapi def json_response(data): - response = requests.Response() - response.status_code = 200 - response._content = data - return response + request = httpx.Request("POST", "https://serpapi.com/image") + return httpx.Response(200, content=data, request=request) def test_upload_image_path_sends_multipart_request(tmp_path): @@ -53,7 +51,7 @@ def test_upload_image_accepts_open_binary_file_and_request_options(): assert result == {"image_id": "image-456"} assert not image.closed - _, request_kwargs = client.session.request.call_args + request_kwargs = client.session.request.call_args.kwargs assert request_kwargs["params"] == {} assert request_kwargs["data"] == { "api_key": "request-api-key", @@ -69,9 +67,10 @@ def test_upload_image_rejects_text_mode_file(tmp_path): client = serpapi.Client(api_key="test-api-key") client.session.request = Mock() - with image_path.open("r") as image: - with pytest.raises(TypeError, match="opened in binary mode"): - client.upload_image(image) + with image_path.open("r") as image, pytest.raises( + TypeError, match="opened in binary mode" + ): + client.upload_image(image) client.session.request.assert_not_called() @@ -92,7 +91,7 @@ def test_request_injects_api_key_when_form_data_does_not_include_it(): client.request("POST", "/example", params={}, data={"field": "value"}) - _, request_kwargs = client.session.request.call_args + request_kwargs = client.session.request.call_args.kwargs assert request_kwargs["params"] == {"api_key": "test-api-key"} assert request_kwargs["data"] == {"field": "value"} diff --git a/tests/test_output_formats.py b/tests/test_output_formats.py index ebafa9d..7acd040 100644 --- a/tests/test_output_formats.py +++ b/tests/test_output_formats.py @@ -1,17 +1,19 @@ from unittest.mock import Mock +import httpx import pytest -import requests import serpapi def response(content_type, content): - response = requests.Response() - response.status_code = 200 - response.headers["Content-Type"] = f"{content_type}; charset=utf-8" - response._content = content.encode("utf-8") - return response + request = httpx.Request("GET", "https://serpapi.com/search") + return httpx.Response( + 200, + headers={"Content-Type": f"{content_type}; charset=utf-8"}, + content=content.encode("utf-8"), + request=request, + ) def test_search_json_returns_serp_results(): diff --git a/tests/test_pagination.py b/tests/test_pagination.py index 6fce28b..53db04e 100644 --- a/tests/test_pagination.py +++ b/tests/test_pagination.py @@ -1,8 +1,8 @@ import json from unittest.mock import Mock +import httpx import pytest -import requests import serpapi @@ -21,10 +21,13 @@ def test_yield_pages_does_not_request_unused_pages( data["serpapi_pagination"] = { "next": f"https://serpapi.com/search?engine=google&q=Coffee&start={page_number * 10}" } - response = requests.Response() - response.status_code = 200 - response.headers["Content-Type"] = "application/json" - response._content = json.dumps(data).encode("utf-8") + request = httpx.Request("GET", "https://serpapi.com/search") + response = httpx.Response( + 200, + headers={"Content-Type": "application/json"}, + content=json.dumps(data).encode("utf-8"), + request=request, + ) responses.append(response) client = serpapi.Client(api_key="test-api-key") diff --git a/tests/test_timeout.py b/tests/test_timeout.py index 7ce5fbd..fb9b0c3 100644 --- a/tests/test_timeout.py +++ b/tests/test_timeout.py @@ -1,39 +1,46 @@ -import pytest -import requests +import httpx + from serpapi import Client + def test_client_timeout_setting(): """Test that timeout can be set on the client and is passed to the request.""" client = Client(api_key="test_key", timeout=10) assert client.timeout == 10 + def test_request_timeout_override(monkeypatch): """Test that timeout can be overridden in the search method.""" client = Client(api_key="test_key", timeout=10) - + def mock_request(method, url, params, headers, timeout, **kwargs): assert timeout == 5 # Return a mock response object - mock_response = requests.Response() - mock_response.status_code = 200 - mock_response._content = b'{"search_metadata": {"id": "123"}}' - return mock_response + request = httpx.Request(method, url) + return httpx.Response( + 200, + content=b'{"search_metadata": {"id": "123"}}', + request=request, + ) monkeypatch.setattr(client.session, "request", mock_request) - + client.search(q="coffee", timeout=5) + def test_request_default_timeout(monkeypatch): """Test that the client's default timeout is used if none is provided in search.""" client = Client(api_key="test_key", timeout=10) - + def mock_request(method, url, params, headers, timeout, **kwargs): assert timeout == 10 - mock_response = requests.Response() - mock_response.status_code = 200 - mock_response._content = b'{"search_metadata": {"id": "123"}}' - return mock_response + request = httpx.Request(method, url) + return httpx.Response( + 200, + content=b'{"search_metadata": {"id": "123"}}', + request=request, + ) monkeypatch.setattr(client.session, "request", mock_request) - + client.search(q="coffee")