diff --git a/CHANGELOG.md b/CHANGELOG.md index 7639471..45e5596 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # Changelog + * [Unreleased] Add Markdown search and archive output support * [2026-02-23] 1.0.3 Enhance error object #16 * [2025-11-17] 1.0.2 Implement `inspect` functions for client #13 * [2025-07-18] 1.0.1 Add support for old Ruby versions (2.7, 3.0) diff --git a/README.md b/README.md index 1943027..fc221af 100644 --- a/README.md +++ b/README.md @@ -55,10 +55,34 @@ client.close - [Asynchronous searches](./demo/demo_async.rb) for submitting non-blocking jobs and retrieving completed results from the Search Archive API. - [Persistent connections and connection pooling](./demo/demo_thread_pool.rb) for reusing HTTP connections across searches. -- JSON responses as Ruby hashes with `search`, or raw search-engine HTML with `html`. +- Search results as Ruby hashes with `search`, token-efficient Markdown with `md`, or raw search-engine HTML with `html`. - SDK methods for the [Location API](https://serpapi.com/locations-api), [Search Archive API](https://serpapi.com/search-archive-api), and [Account API](https://serpapi.com/account-api). - Configurable HTTP timeouts and symbolized or string JSON keys. +## Response formats + +Use `search` for structured results decoded into a Ruby `Hash`: + +```ruby +results = client.search(q: "coffee") +``` + +Use `md` for a token-efficient Markdown `String` optimized for LLMs and AI agents: + +```ruby +markdown = client.md(q: "coffee") +``` + +Use `html` when you need the raw search-engine response: + +```ruby +raw_html = client.html(q: "coffee") +``` + +Archived results are also available as Markdown with `client.search_archive(search_id, :md)`. + +Learn more about [SerpApi Markdown output](https://serpapi.com/markdown-output). + ## Configuration Set defaults when creating a client, then override search parameters in individual calls: diff --git a/README.md.erb b/README.md.erb index 200bc5d..29a9f83 100644 --- a/README.md.erb +++ b/README.md.erb @@ -55,10 +55,34 @@ client.close - [Asynchronous searches](./demo/demo_async.rb) for submitting non-blocking jobs and retrieving completed results from the Search Archive API. - [Persistent connections and connection pooling](./demo/demo_thread_pool.rb) for reusing HTTP connections across searches. -- JSON responses as Ruby hashes with `search`, or raw search-engine HTML with `html`. +- Search results as Ruby hashes with `search`, token-efficient Markdown with `md`, or raw search-engine HTML with `html`. - SDK methods for the [Location API](https://serpapi.com/locations-api), [Search Archive API](https://serpapi.com/search-archive-api), and [Account API](https://serpapi.com/account-api). - Configurable HTTP timeouts and symbolized or string JSON keys. +## Response formats + +Use `search` for structured results decoded into a Ruby `Hash`: + +```ruby +results = client.search(q: "coffee") +``` + +Use `md` for a token-efficient Markdown `String` optimized for LLMs and AI agents: + +```ruby +markdown = client.md(q: "coffee") +``` + +Use `html` when you need the raw search-engine response: + +```ruby +raw_html = client.html(q: "coffee") +``` + +Archived results are also available as Markdown with `client.search_archive(search_id, :md)`. + +Learn more about [SerpApi Markdown output](https://serpapi.com/markdown-output). + ## Configuration Set defaults when creating a client, then override search parameters in individual calls: diff --git a/lib/serpapi/client.rb b/lib/serpapi/client.rb index 0a8f081..8f820ff 100644 --- a/lib/serpapi/client.rb +++ b/lib/serpapi/client.rb @@ -7,7 +7,7 @@ module SerpApi # features: # * async non-block search # * persistent HTTP connection - # * search API + # * search API with JSON, HTML, and Markdown output # * location API # * account API # * search archive API @@ -15,6 +15,9 @@ module SerpApi class Client # Backend service URL BACKEND = 'serpapi.com'.freeze + # SerpApi errors are JSON even when HTML or Markdown output was requested. + # Decode the actual Content-Type so structured errors and successful JSON responses become Hashes. + CONTENT_TYPE_DECODERS = { 'application/json' => :json, 'text/html' => :html, 'text/markdown' => :md }.freeze # HTTP timeout requests attr_reader :timeout, @@ -108,7 +111,7 @@ def initialize(params = {}) # thus, most of the compute power is on the backsdend and not on the client side. # @param [Hash] params includes engine, api_key, search fields and more.. # this override the default params provided to the constructor. - # @return [Hash] search results formatted as a Hash. + # @return [Hash|String] search results formatted as a Hash or raw text. def search(params = {}) get('/search', :json, params) end @@ -118,11 +121,22 @@ def search(params = {}) # it is useful for training AI models, RAG, debugging # or when you need to parse the HTML yourself. # - # @return [String] raw html search results directly from the search engine. + # @return [String] raw HTML search results. def html(params = {}) + params = params.reject { |key, _| key.to_s == 'output' }.merge(output: 'html') if params.instance_of?(Hash) get('/search', :html, params) end + # Perform a search using SerpApi.com and return results optimized for LLMs and AI agents. + # The output contains Markdown tables, links, and YAML frontmatter. + # + # @param [Hash] params includes engine, api_key, search fields and more. + # @return [String] search results formatted as Markdown. + def md(params = {}) + params = params.reject { |key, _| key.to_s == 'output' }.merge(output: 'md') if params.instance_of?(Hash) + get('/search', :md, params) + end + # Get location using Location API # # example: spec/serpapi/location_api_spec.rb @@ -146,10 +160,10 @@ def location(params = {}) # doc: https://serpapi.com/search-archive-api # # @param [String|Integer] search_id from original search `results[:search_metadata][:id]` - # @param [Symbol] format :json or :html [default: json, optional] - # @return [String|Hash] raw html or JSON / Hash + # @param [Symbol] format :json, :html, or :md [default: json, optional] + # @return [String|Hash] raw HTML, Markdown, or JSON / Hash def search_archive(search_id, format = :json) - raise SerpApiError, 'format must be json or html' unless [:json, :html].include?(format) + raise SerpApiError, 'format must be json, html, or md' unless [:json, :html, :md].include?(format) get("/searches/#{search_id}.#{format}", format) end @@ -195,6 +209,8 @@ def query(params) # merge default params with custom params q = @params.clone.merge(params) + q.delete('output') if params.key?(:output) + q.delete(:output) if params.key?('output') && !params.key?(:output) # do not pollute default params with custom params q.delete(:symbolize_names) if q.key?(:symbolize_names) @@ -211,12 +227,14 @@ def persistent? # Perform HTTP GET request to the SerpApi.com backend endpoint. # # @param [String] endpoint HTTP service URI - # @param [Symbol] decoder type :json or :html + # @param [Symbol] decoder type :json, :html, or :md # @param [Hash] params custom search inputs - # @return [String|Hash] raw HTML or decoded response as JSON / Hash + # @return [String|Hash] raw text or decoded response as JSON / Hash def get(endpoint, decoder = :json, params = {}) response = execute_request(endpoint, params) - handle_response(response, decoder, endpoint, params) + handle_response(response, response_decoder(response, decoder), endpoint, params) + ensure + response&.flush if persistent? end def execute_request(endpoint, params) @@ -228,14 +246,19 @@ def execute_request(endpoint, params) end end + def response_decoder(response, default) + content_type = response.headers['Content-Type'].to_s.split(';').first + CONTENT_TYPE_DECODERS.fetch(content_type, default) + end + def handle_response(response, decoder, endpoint, params) case decoder when :json process_json_response(response, endpoint, params) - when :html - process_html_response(response, endpoint, params) + when :html, :md + process_text_response(response, endpoint, params, decoder) else - raise SerpApiError, "not supported decoder: #{decoder}, available: :json, :html" + raise SerpApiError, "not supported decoder: #{decoder}, available: :json, :html, :md" end end @@ -249,13 +272,13 @@ def process_json_response(response, endpoint, params) raise_parser_error(response, endpoint, params) end - response.flush if persistent? data end - def process_html_response(response, endpoint, params) - raise_http_error(response, nil, endpoint, params, decoder: :html) if response.status != 200 - response.body + def process_text_response(response, endpoint, params, decoder) + raise_http_error(response, nil, endpoint, params, decoder: decoder) if response.status != 200 + + response.body.to_s end def validate_json_content!(data, response, endpoint, params) diff --git a/lib/serpapi/error.rb b/lib/serpapi/error.rb index b082178..f2c3fea 100644 --- a/lib/serpapi/error.rb +++ b/lib/serpapi/error.rb @@ -10,7 +10,7 @@ module SerpApi # - search_params: Hash of search parameters used (optional) # - response_status: Integer HTTP or response status code (optional) # - search_id: String id returned by the service for the search (optional) - # - decoder: Symbol representing the decoder/format used (optional) (e.g. :json) + # - decoder: Symbol representing the decoder/format used (optional) (e.g. :json or :md) class SerpApiError < StandardError attr_reader :serpapi_error, :search_params, :response_status, :search_id, :decoder @@ -21,7 +21,7 @@ class SerpApiError < StandardError # @param search_params [Hash, nil] optional hash of the search parameters used # @param response_status [Integer, nil] optional HTTP or response status code # @param search_id [String, nil] optional id returned by the service for the search - # @param decoder [Symbol, nil] optional decoder/format used (e.g. :json) + # @param decoder [Symbol, nil] optional decoder/format used (e.g. :json or :md) def initialize(message = nil, serpapi_error: nil, search_params: nil, diff --git a/spec/serpapi/client/client_spec.rb b/spec/serpapi/client/client_spec.rb index d3c2736..8384d3b 100644 --- a/spec/serpapi/client/client_spec.rb +++ b/spec/serpapi/client/client_spec.rb @@ -25,9 +25,49 @@ expect(results.keys).to include('search_metadata'), 'search_metadata should be present in the results' end + it 'selects the decoder from the output parameter' do + json = client.search(q: 'Coffee', location: 'Austin, TX', output: 'json') + html = client.search(q: 'Coffee', location: 'Austin, TX', output: 'html') + markdown = client.search(q: 'Coffee', location: 'Austin, TX', output: 'md') + + expect(json).to be_a(Hash) + expect(html).to match(/coffee/i) + expect(markdown).to be_a(String) + expect(markdown).to start_with('---') + end + + it 'ignores output parameters passed to format shortcut methods' do + html = client.html(q: 'Coffee', location: 'Austin, TX', output: 'json') + markdown = client.md(q: 'Coffee', location: 'Austin, TX', 'output' => 'html') + + expect(html).to match(/\A/i) + expect(markdown).to start_with('---') + end + it 'search for coffee in Austin, TX and receive raw HTML' do results = client.html(q: 'Coffee', location: 'Austin, TX') - expect(results).to match(/coffee/i) + + expect(results).to be_a(String) + expect(results).to match(/\A/i) + end + + it 'search for coffee in Austin, TX and receive Markdown' do + results = client.md(q: 'Coffee', location: 'Austin, TX') + + expect(results).to be_a(String) + expect(results).to start_with('---') + expect(results).to include('## Organic Results') + end + + it 'decodes JSON errors and reuses the persistent connection' do + expect { + client.md + }.to raise_error(SerpApi::SerpApiError) do |error| + expect(error.decoder).to eq(:json) + expect(error.serpapi_error).to include('Missing query') + end + + expect(client.md(q: 'Coffee')).to start_with('---') end it 'missing query' do @@ -80,7 +120,7 @@ begin client.send(:get, '/invalid', :json, {}) rescue SerpApi::SerpApiError => e - expect(e.message).to include('JSON parse error') + expect(e.message).to include('HTTP request failed with status: 404') rescue => e raise("wrong exception: #{e}") end diff --git a/spec/serpapi/client/search_archive_api_spec.rb b/spec/serpapi/client/search_archive_api_spec.rb index 0ecc422..2586653 100644 --- a/spec/serpapi/client/search_archive_api_spec.rb +++ b/spec/serpapi/client/search_archive_api_spec.rb @@ -26,6 +26,10 @@ client = SerpApi::Client.new(api_key: client.api_key, engine: 'google') results = client.search_archive(search_id) expect(archive_search).to eq(results) + + markdown = client.search_archive(search_id, :md) + expect(markdown).to be_a(String) + expect(markdown).to start_with('---') else client = SerpApi::Client.new(api_key: client.api_key, engine: 'google') allow(client).to receive(:get) { search_response_mock }