diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 64f9ff4..427b8ec 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "2.8.0" + ".": "2.9.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index 74e73bc..97db86d 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ -configured_endpoints: 38 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-155e4761d62255841349c0f8a01b0a9c463ea1d1f2d6c4fd8d1a75c8bef6f226.yml -openapi_spec_hash: ab91f77e7c9d992400cbc7fc8a9e76c1 -config_hash: bff282047fafdad771fb7ec685f56944 +configured_endpoints: 40 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-a0dda03bbb600917cfb9add468cc4c8c84351a8dbbf61644dbc353263ca1748f.yml +openapi_spec_hash: c24264f32a46d9317aac5af9d6a396f7 +config_hash: 920678668dd2da6f8966fbf1b8fde4e2 diff --git a/CHANGELOG.md b/CHANGELOG.md index cabb035..9e5e281 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 2.9.0 (2026-08-07) + +Full Changelog: [v2.8.0...v2.9.0](https://github.com/context-dot-dev/context-python-sdk/compare/v2.8.0...v2.9.0) + +### Features + +* **api:** api update ([b2b2312](https://github.com/context-dot-dev/context-python-sdk/commit/b2b2312efd4957bb5323718bcdc2cc1bb709e667)) +* **api:** api update ([040f31d](https://github.com/context-dot-dev/context-python-sdk/commit/040f31d9ff3c297156b913be31d77a39046ccd5b)) +* **api:** api update ([cbf9c43](https://github.com/context-dot-dev/context-python-sdk/commit/cbf9c43bd14c3eb845ff45bf4e6e0e7c18fae5aa)) +* **api:** api update ([06c5d6c](https://github.com/context-dot-dev/context-python-sdk/commit/06c5d6cb6e41c9557eac78249da5460f999e12c6)) +* **api:** api update ([3935a16](https://github.com/context-dot-dev/context-python-sdk/commit/3935a1695972bbe75c7035ec89efd2877be71f90)) +* **api:** api update ([6f0f3fb](https://github.com/context-dot-dev/context-python-sdk/commit/6f0f3fbc0966c2b69a4fa594204d6cb8b115ec60)) +* **api:** api update ([d9a026d](https://github.com/context-dot-dev/context-python-sdk/commit/d9a026d6526e93e54ec2fcddc7ee7988748370c8)) +* **api:** manual updates ([a62c8a4](https://github.com/context-dot-dev/context-python-sdk/commit/a62c8a49822503dbc13baa8258476b732c2b4e43)) + ## 2.8.0 (2026-08-05) Full Changelog: [v2.7.0...v2.8.0](https://github.com/context-dot-dev/context-python-sdk/compare/v2.7.0...v2.8.0) diff --git a/api.md b/api.md index 61a5c97..56ac7f5 100644 --- a/api.md +++ b/api.md @@ -151,6 +151,7 @@ from context.dev.types import ( Intake, BatchRetrieveResponse, BatchListResponse, + BatchDeleteResponse, BatchCancelResponse, BatchGetResultsResponse, BatchSubmitResponse, @@ -161,6 +162,19 @@ Methods: - client.batch.retrieve(batch_id) -> BatchRetrieveResponse - client.batch.list(\*\*params) -> BatchListResponse +- client.batch.delete(batch_id) -> BatchDeleteResponse - client.batch.cancel(batch_id) -> BatchCancelResponse - client.batch.get_results(batch_id, \*\*params) -> BatchGetResultsResponse -- client.batch.submit(\*\*params) -> BatchSubmitResponse +- client.batch.submit(\*\*params) -> BatchSubmitResponse + +# People + +Types: + +```python +from context.dev.types import PersonEnrichResponse +``` + +Methods: + +- client.people.enrich(\*\*params) -> PersonEnrichResponse diff --git a/pyproject.toml b/pyproject.toml index ece6698..7c3d73f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "context.dev" -version = "2.8.0" +version = "2.9.0" description = "The official Python library for the context.dev API" dynamic = ["readme"] license = "Apache-2.0" diff --git a/src/context/dev/_client.py b/src/context/dev/_client.py index cf4d545..63888b2 100644 --- a/src/context/dev/_client.py +++ b/src/context/dev/_client.py @@ -35,12 +35,13 @@ ) if TYPE_CHECKING: - from .resources import ai, web, batch, brand, parse, utility, industry, monitors + from .resources import ai, web, batch, brand, parse, people, utility, industry, monitors from .resources.ai import AIResource, AsyncAIResource from .resources.web import WebResource, AsyncWebResource from .resources.batch import BatchResource, AsyncBatchResource from .resources.brand import BrandResource, AsyncBrandResource from .resources.parse import ParseResource, AsyncParseResource + from .resources.people import PeopleResource, AsyncPeopleResource from .resources.utility import UtilityResource, AsyncUtilityResource from .resources.industry import IndustryResource, AsyncIndustryResource from .resources.monitors import MonitorsResource, AsyncMonitorsResource @@ -168,10 +169,17 @@ def monitors(self) -> MonitorsResource: @cached_property def batch(self) -> BatchResource: + """Scrape many pages or crawl a site asynchronously.""" from .resources.batch import BatchResource return BatchResource(self) + @cached_property + def people(self) -> PeopleResource: + from .resources.people import PeopleResource + + return PeopleResource(self) + @cached_property def with_raw_response(self) -> ContextDevWithRawResponse: return ContextDevWithRawResponse(self) @@ -396,10 +404,17 @@ def monitors(self) -> AsyncMonitorsResource: @cached_property def batch(self) -> AsyncBatchResource: + """Scrape many pages or crawl a site asynchronously.""" from .resources.batch import AsyncBatchResource return AsyncBatchResource(self) + @cached_property + def people(self) -> AsyncPeopleResource: + from .resources.people import AsyncPeopleResource + + return AsyncPeopleResource(self) + @cached_property def with_raw_response(self) -> AsyncContextDevWithRawResponse: return AsyncContextDevWithRawResponse(self) @@ -566,10 +581,17 @@ def monitors(self) -> monitors.MonitorsResourceWithRawResponse: @cached_property def batch(self) -> batch.BatchResourceWithRawResponse: + """Scrape many pages or crawl a site asynchronously.""" from .resources.batch import BatchResourceWithRawResponse return BatchResourceWithRawResponse(self._client.batch) + @cached_property + def people(self) -> people.PeopleResourceWithRawResponse: + from .resources.people import PeopleResourceWithRawResponse + + return PeopleResourceWithRawResponse(self._client.people) + class AsyncContextDevWithRawResponse: _client: AsyncContextDev @@ -624,10 +646,17 @@ def monitors(self) -> monitors.AsyncMonitorsResourceWithRawResponse: @cached_property def batch(self) -> batch.AsyncBatchResourceWithRawResponse: + """Scrape many pages or crawl a site asynchronously.""" from .resources.batch import AsyncBatchResourceWithRawResponse return AsyncBatchResourceWithRawResponse(self._client.batch) + @cached_property + def people(self) -> people.AsyncPeopleResourceWithRawResponse: + from .resources.people import AsyncPeopleResourceWithRawResponse + + return AsyncPeopleResourceWithRawResponse(self._client.people) + class ContextDevWithStreamedResponse: _client: ContextDev @@ -682,10 +711,17 @@ def monitors(self) -> monitors.MonitorsResourceWithStreamingResponse: @cached_property def batch(self) -> batch.BatchResourceWithStreamingResponse: + """Scrape many pages or crawl a site asynchronously.""" from .resources.batch import BatchResourceWithStreamingResponse return BatchResourceWithStreamingResponse(self._client.batch) + @cached_property + def people(self) -> people.PeopleResourceWithStreamingResponse: + from .resources.people import PeopleResourceWithStreamingResponse + + return PeopleResourceWithStreamingResponse(self._client.people) + class AsyncContextDevWithStreamedResponse: _client: AsyncContextDev @@ -740,10 +776,17 @@ def monitors(self) -> monitors.AsyncMonitorsResourceWithStreamingResponse: @cached_property def batch(self) -> batch.AsyncBatchResourceWithStreamingResponse: + """Scrape many pages or crawl a site asynchronously.""" from .resources.batch import AsyncBatchResourceWithStreamingResponse return AsyncBatchResourceWithStreamingResponse(self._client.batch) + @cached_property + def people(self) -> people.AsyncPeopleResourceWithStreamingResponse: + from .resources.people import AsyncPeopleResourceWithStreamingResponse + + return AsyncPeopleResourceWithStreamingResponse(self._client.people) + Client = ContextDev diff --git a/src/context/dev/_version.py b/src/context/dev/_version.py index c08edfc..593f642 100644 --- a/src/context/dev/_version.py +++ b/src/context/dev/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. __title__ = "context.dev" -__version__ = "2.8.0" # x-release-please-version +__version__ = "2.9.0" # x-release-please-version diff --git a/src/context/dev/resources/__init__.py b/src/context/dev/resources/__init__.py index c99c0f9..9d68085 100644 --- a/src/context/dev/resources/__init__.py +++ b/src/context/dev/resources/__init__.py @@ -40,6 +40,14 @@ ParseResourceWithStreamingResponse, AsyncParseResourceWithStreamingResponse, ) +from .people import ( + PeopleResource, + AsyncPeopleResource, + PeopleResourceWithRawResponse, + AsyncPeopleResourceWithRawResponse, + PeopleResourceWithStreamingResponse, + AsyncPeopleResourceWithStreamingResponse, +) from .utility import ( UtilityResource, AsyncUtilityResource, @@ -114,4 +122,10 @@ "AsyncBatchResourceWithRawResponse", "BatchResourceWithStreamingResponse", "AsyncBatchResourceWithStreamingResponse", + "PeopleResource", + "AsyncPeopleResource", + "PeopleResourceWithRawResponse", + "AsyncPeopleResourceWithRawResponse", + "PeopleResourceWithStreamingResponse", + "AsyncPeopleResourceWithStreamingResponse", ] diff --git a/src/context/dev/resources/batch.py b/src/context/dev/resources/batch.py index 0244b3e..2b1406c 100644 --- a/src/context/dev/resources/batch.py +++ b/src/context/dev/resources/batch.py @@ -8,7 +8,7 @@ from ..types import batch_list_params, batch_submit_params, batch_get_results_params from .._types import Body, Omit, Query, Headers, NotGiven, SequenceNotStr, omit, not_given -from .._utils import path_template, maybe_transform, async_maybe_transform +from .._utils import path_template, maybe_transform, strip_not_given, async_maybe_transform from .._compat import cached_property from .._resource import SyncAPIResource, AsyncAPIResource from .._response import ( @@ -20,6 +20,7 @@ from .._base_client import make_request_options from ..types.batch_list_response import BatchListResponse from ..types.batch_cancel_response import BatchCancelResponse +from ..types.batch_delete_response import BatchDeleteResponse from ..types.batch_submit_response import BatchSubmitResponse from ..types.batch_retrieve_response import BatchRetrieveResponse from ..types.batch_get_results_response import BatchGetResultsResponse @@ -28,6 +29,8 @@ class BatchResource(SyncAPIResource): + """Scrape many pages or crawl a site asynchronously.""" + @cached_property def with_raw_response(self) -> BatchResourceWithRawResponse: """ @@ -148,6 +151,43 @@ def list( cast_to=BatchListResponse, ) + def delete( + self, + batch_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> BatchDeleteResponse: + """Permanently delete a finished batch and its stored results. + + Active batches must + settle first. + + Args: + batch_id: ID of the batch to retrieve or cancel. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not batch_id: + raise ValueError(f"Expected a non-empty value for `batch_id` but received {batch_id!r}") + return self._delete( + path_template("/batch/{batch_id}", batch_id=batch_id), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=BatchDeleteResponse, + ) + def cancel( self, batch_id: str, @@ -241,9 +281,10 @@ def get_results( def submit( self, *, - identifiers: batch_submit_params.Identifiers, + input: batch_submit_params.Input, tags: SequenceNotStr[str] | Omit = omit, - timeout_ms: int | Omit = omit, + webhook_url: str | Omit = omit, + idempotency_key: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -252,16 +293,17 @@ def submit( timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> BatchSubmitResponse: """ - Retrieve and normalize a person profile from identifiers. + Scrape 25K URLs or crawl large websites asynchronously. Args: - identifiers: Known identifiers for the person. At least one identifier is required. + input: Choose a URL list or a site crawl. + + tags: Tags stored on the batch. Filter the batch list by them later. - tags: Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters. + webhook_url: URL notified when the batch finishes. - timeout_ms: Optional timeout in milliseconds for the request. If the request takes longer - than this value, it will be aborted with a 408 status code. Maximum allowed - value is 300000ms (5 minutes). + idempotency_key: Any string unique to this submission. Retries with the same key return the + original batch. extra_headers: Send extra headers @@ -271,13 +313,14 @@ def submit( timeout: Override the client-level default timeout for this request, in seconds """ + extra_headers = {**strip_not_given({"Idempotency-Key": idempotency_key}), **(extra_headers or {})} return self._post( - "/people/retrieve", + "/batch/submit", body=maybe_transform( { - "identifiers": identifiers, + "input": input, "tags": tags, - "timeout_ms": timeout_ms, + "webhook_url": webhook_url, }, batch_submit_params.BatchSubmitParams, ), @@ -289,6 +332,8 @@ def submit( class AsyncBatchResource(AsyncAPIResource): + """Scrape many pages or crawl a site asynchronously.""" + @cached_property def with_raw_response(self) -> AsyncBatchResourceWithRawResponse: """ @@ -409,6 +454,43 @@ async def list( cast_to=BatchListResponse, ) + async def delete( + self, + batch_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> BatchDeleteResponse: + """Permanently delete a finished batch and its stored results. + + Active batches must + settle first. + + Args: + batch_id: ID of the batch to retrieve or cancel. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not batch_id: + raise ValueError(f"Expected a non-empty value for `batch_id` but received {batch_id!r}") + return await self._delete( + path_template("/batch/{batch_id}", batch_id=batch_id), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=BatchDeleteResponse, + ) + async def cancel( self, batch_id: str, @@ -502,9 +584,10 @@ async def get_results( async def submit( self, *, - identifiers: batch_submit_params.Identifiers, + input: batch_submit_params.Input, tags: SequenceNotStr[str] | Omit = omit, - timeout_ms: int | Omit = omit, + webhook_url: str | Omit = omit, + idempotency_key: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -513,16 +596,17 @@ async def submit( timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> BatchSubmitResponse: """ - Retrieve and normalize a person profile from identifiers. + Scrape 25K URLs or crawl large websites asynchronously. Args: - identifiers: Known identifiers for the person. At least one identifier is required. + input: Choose a URL list or a site crawl. + + tags: Tags stored on the batch. Filter the batch list by them later. - tags: Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters. + webhook_url: URL notified when the batch finishes. - timeout_ms: Optional timeout in milliseconds for the request. If the request takes longer - than this value, it will be aborted with a 408 status code. Maximum allowed - value is 300000ms (5 minutes). + idempotency_key: Any string unique to this submission. Retries with the same key return the + original batch. extra_headers: Send extra headers @@ -532,13 +616,14 @@ async def submit( timeout: Override the client-level default timeout for this request, in seconds """ + extra_headers = {**strip_not_given({"Idempotency-Key": idempotency_key}), **(extra_headers or {})} return await self._post( - "/people/retrieve", + "/batch/submit", body=await async_maybe_transform( { - "identifiers": identifiers, + "input": input, "tags": tags, - "timeout_ms": timeout_ms, + "webhook_url": webhook_url, }, batch_submit_params.BatchSubmitParams, ), @@ -559,6 +644,9 @@ def __init__(self, batch: BatchResource) -> None: self.list = to_raw_response_wrapper( batch.list, ) + self.delete = to_raw_response_wrapper( + batch.delete, + ) self.cancel = to_raw_response_wrapper( batch.cancel, ) @@ -580,6 +668,9 @@ def __init__(self, batch: AsyncBatchResource) -> None: self.list = async_to_raw_response_wrapper( batch.list, ) + self.delete = async_to_raw_response_wrapper( + batch.delete, + ) self.cancel = async_to_raw_response_wrapper( batch.cancel, ) @@ -601,6 +692,9 @@ def __init__(self, batch: BatchResource) -> None: self.list = to_streamed_response_wrapper( batch.list, ) + self.delete = to_streamed_response_wrapper( + batch.delete, + ) self.cancel = to_streamed_response_wrapper( batch.cancel, ) @@ -622,6 +716,9 @@ def __init__(self, batch: AsyncBatchResource) -> None: self.list = async_to_streamed_response_wrapper( batch.list, ) + self.delete = async_to_streamed_response_wrapper( + batch.delete, + ) self.cancel = async_to_streamed_response_wrapper( batch.cancel, ) diff --git a/src/context/dev/resources/brand.py b/src/context/dev/resources/brand.py index cf8f895..b7bb4ca 100644 --- a/src/context/dev/resources/brand.py +++ b/src/context/dev/resources/brand.py @@ -1288,7 +1288,8 @@ def search( ) -> BrandSearchResponse: """ Search brands by name or domain and get back up to 10 lightweight matches - (domain, name, logo), most popular first: by Tranco rank, then market cap for + (domain, name, logo). Name matches rank ahead of domain matches; within each + group the most popular brands come first: by Tranco rank, then market cap for brands outside the Tranco list, with text relevance breaking ties. Matching is prefix-based with no typo tolerance, so it is suited to autocomplete. Only brands already in the Context.dev index are returned — use /brand/retrieve to @@ -2592,7 +2593,8 @@ async def search( ) -> BrandSearchResponse: """ Search brands by name or domain and get back up to 10 lightweight matches - (domain, name, logo), most popular first: by Tranco rank, then market cap for + (domain, name, logo). Name matches rank ahead of domain matches; within each + group the most popular brands come first: by Tranco rank, then market cap for brands outside the Tranco list, with text relevance breaking ties. Matching is prefix-based with no typo tolerance, so it is suited to autocomplete. Only brands already in the Context.dev index are returned — use /brand/retrieve to diff --git a/src/context/dev/resources/parse.py b/src/context/dev/resources/parse.py index 12c0b8f..3c3101a 100644 --- a/src/context/dev/resources/parse.py +++ b/src/context/dev/resources/parse.py @@ -165,10 +165,11 @@ def handle( include_links: Preserve hyperlinks in Markdown output - ocr: When true for PDF inputs, detect and OCR images embedded in the selected pages, - inserting recognized text at each image's position in page reading order while - preserving the PDF text layer. pdf.start/pdf.end limit the inclusive page range. - When false, all OCR is disabled, including the automatic scanned-PDF fallback. + ocr: When true for PDF inputs, OCR the selected pages that have no usable text layer + (scans), replacing each recovered page's text with the OCR result while pages + with a real text layer keep it. pdf.start/pdf.end limit the inclusive page + range. Billed at 1 credit per page OCR actually recovered, on top of the base + request cost. When false, no OCR runs. pdf: PDF page-range options as a JSON object, e.g. {"start": 2, "end": 5}. @@ -349,10 +350,11 @@ async def handle( include_links: Preserve hyperlinks in Markdown output - ocr: When true for PDF inputs, detect and OCR images embedded in the selected pages, - inserting recognized text at each image's position in page reading order while - preserving the PDF text layer. pdf.start/pdf.end limit the inclusive page range. - When false, all OCR is disabled, including the automatic scanned-PDF fallback. + ocr: When true for PDF inputs, OCR the selected pages that have no usable text layer + (scans), replacing each recovered page's text with the OCR result while pages + with a real text layer keep it. pdf.start/pdf.end limit the inclusive page + range. Billed at 1 credit per page OCR actually recovered, on top of the base + request cost. When false, no OCR runs. pdf: PDF page-range options as a JSON object, e.g. {"start": 2, "end": 5}. diff --git a/src/context/dev/resources/people.py b/src/context/dev/resources/people.py new file mode 100644 index 0000000..fa683db --- /dev/null +++ b/src/context/dev/resources/people.py @@ -0,0 +1,221 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Iterable + +import httpx + +from ..types import person_enrich_params +from .._types import Body, Omit, Query, Headers, NotGiven, SequenceNotStr, omit, not_given +from .._utils import maybe_transform, async_maybe_transform +from .._compat import cached_property +from .._resource import SyncAPIResource, AsyncAPIResource +from .._response import ( + to_raw_response_wrapper, + to_streamed_response_wrapper, + async_to_raw_response_wrapper, + async_to_streamed_response_wrapper, +) +from .._base_client import make_request_options +from ..types.person_enrich_response import PersonEnrichResponse + +__all__ = ["PeopleResource", "AsyncPeopleResource"] + + +class PeopleResource(SyncAPIResource): + @cached_property + def with_raw_response(self) -> PeopleResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/context-dot-dev/context-python-sdk#accessing-raw-response-data-eg-headers + """ + return PeopleResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> PeopleResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/context-dot-dev/context-python-sdk#with_streaming_response + """ + return PeopleResourceWithStreamingResponse(self) + + def enrich( + self, + *, + company: person_enrich_params.Company | Omit = omit, + education: Iterable[person_enrich_params.Education] | Omit = omit, + email: str | Omit = omit, + location: person_enrich_params.Location | Omit = omit, + name: person_enrich_params.Name | Omit = omit, + social_urls: SequenceNotStr[str] | Omit = omit, + tags: SequenceNotStr[str] | Omit = omit, + timeout_ms: int | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> PersonEnrichResponse: + """ + Finds and normalizes the best available person candidate from additive identity + clues, then assigns an identity match score from 0 to 100. Available on all paid + plans. Successful requests cost 20 credits. Disposable and free email addresses + (like gmail.com, yahoo.com) will throw a 422 error. + + Args: + tags: Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters. + + timeout_ms: Optional timeout in milliseconds for the request. If the request takes longer + than this value, it will be aborted with a 408 status code. Maximum allowed + value is 300000ms (5 minutes). + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._post( + "/people/enrich", + body=maybe_transform( + { + "company": company, + "education": education, + "email": email, + "location": location, + "name": name, + "social_urls": social_urls, + "tags": tags, + "timeout_ms": timeout_ms, + }, + person_enrich_params.PersonEnrichParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=PersonEnrichResponse, + ) + + +class AsyncPeopleResource(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncPeopleResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/context-dot-dev/context-python-sdk#accessing-raw-response-data-eg-headers + """ + return AsyncPeopleResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncPeopleResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/context-dot-dev/context-python-sdk#with_streaming_response + """ + return AsyncPeopleResourceWithStreamingResponse(self) + + async def enrich( + self, + *, + company: person_enrich_params.Company | Omit = omit, + education: Iterable[person_enrich_params.Education] | Omit = omit, + email: str | Omit = omit, + location: person_enrich_params.Location | Omit = omit, + name: person_enrich_params.Name | Omit = omit, + social_urls: SequenceNotStr[str] | Omit = omit, + tags: SequenceNotStr[str] | Omit = omit, + timeout_ms: int | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> PersonEnrichResponse: + """ + Finds and normalizes the best available person candidate from additive identity + clues, then assigns an identity match score from 0 to 100. Available on all paid + plans. Successful requests cost 20 credits. Disposable and free email addresses + (like gmail.com, yahoo.com) will throw a 422 error. + + Args: + tags: Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters. + + timeout_ms: Optional timeout in milliseconds for the request. If the request takes longer + than this value, it will be aborted with a 408 status code. Maximum allowed + value is 300000ms (5 minutes). + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return await self._post( + "/people/enrich", + body=await async_maybe_transform( + { + "company": company, + "education": education, + "email": email, + "location": location, + "name": name, + "social_urls": social_urls, + "tags": tags, + "timeout_ms": timeout_ms, + }, + person_enrich_params.PersonEnrichParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=PersonEnrichResponse, + ) + + +class PeopleResourceWithRawResponse: + def __init__(self, people: PeopleResource) -> None: + self._people = people + + self.enrich = to_raw_response_wrapper( + people.enrich, + ) + + +class AsyncPeopleResourceWithRawResponse: + def __init__(self, people: AsyncPeopleResource) -> None: + self._people = people + + self.enrich = async_to_raw_response_wrapper( + people.enrich, + ) + + +class PeopleResourceWithStreamingResponse: + def __init__(self, people: PeopleResource) -> None: + self._people = people + + self.enrich = to_streamed_response_wrapper( + people.enrich, + ) + + +class AsyncPeopleResourceWithStreamingResponse: + def __init__(self, people: AsyncPeopleResource) -> None: + self._people = people + + self.enrich = async_to_streamed_response_wrapper( + people.enrich, + ) diff --git a/src/context/dev/resources/web.py b/src/context/dev/resources/web.py index 889ed6d..efe42f7 100644 --- a/src/context/dev/resources/web.py +++ b/src/context/dev/resources/web.py @@ -2047,6 +2047,18 @@ def web_scrape_md( responses from a recognized API key; use error_code to distinguish stable failure categories. + ### YouTube + + YouTube URLs return the video or channel itself rather than the surrounding + player and navigation chrome. A URL addressing a single video (`/watch`, + `youtu.be`, `/shorts`, `/embed`, `/live`) returns its title, channel, duration, + view count, keywords, full description, and the transcript when the video has + captions that can be retrieved; videos without captions return everything except + the transcript. A channel URL (`/channel/UC…`, `/@handle`, `/c/…`, `/user/…`) + returns its name, handle, subscriber count, video count, and full description. + When `includeImages=true`, video responses also include the thumbnail and + channel responses include the avatar. Costs the same as any other scrape. + ### Billing & errors | HTTP status | Billed? | Meaning | @@ -2056,6 +2068,7 @@ def web_scrape_md( | 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code | | 404 | No | Target page returned or fingerprinted as not found | | 408 | No | Request timed out | + | 413 | No | Target content exceeds the maximum supported size (20 MB) | | 415 | No | Unsupported content type | | 429 | No | Per-minute rate limit exceeded; honor Retry-After | | 500 | No | Internal error | @@ -2169,6 +2182,7 @@ def web_scrape_sitemap( domain: str, headers: Dict[str, str] | Omit = omit, max_links: int | Omit = omit, + search: str | Omit = omit, sitemap_url: str | Omit = omit, tags: SequenceNotStr[str] | Omit = omit, timeout_ms: int | Omit = omit, @@ -2181,8 +2195,13 @@ def web_scrape_sitemap( extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> WebWebScrapeSitemapResponse: - """ - Crawl an entire website's sitemap and return all discovered page URLs. + """Crawl an entire website's sitemap and return all discovered page URLs. + + Pass + `search` to have the crawled sitemap filtered down to the pages about a phrase + (for example `pricing and plans` or `api authentication docs`), most relevant + first — a searched crawl scans the whole sitemap and costs 2 credits instead + of 1. Args: domain: Domain to build a sitemap for @@ -2194,6 +2213,10 @@ def web_scrape_sitemap( max_links: Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Minimum is 1, maximum is 100,000. + search: Optional search phrase. When provided, the crawled sitemap is filtered to the + pages whose URLs are about that phrase, most relevant first, and the request + costs 2 credits instead of 1. + sitemap_url: Optional explicit sitemap URL. When provided, exactly this sitemap is crawled instead of discovering the domain's sitemaps. @@ -2233,6 +2256,7 @@ def web_scrape_sitemap( "domain": domain, "headers": headers, "max_links": max_links, + "search": search, "sitemap_url": sitemap_url, "tags": tags, "timeout_ms": timeout_ms, @@ -4247,6 +4271,18 @@ async def web_scrape_md( responses from a recognized API key; use error_code to distinguish stable failure categories. + ### YouTube + + YouTube URLs return the video or channel itself rather than the surrounding + player and navigation chrome. A URL addressing a single video (`/watch`, + `youtu.be`, `/shorts`, `/embed`, `/live`) returns its title, channel, duration, + view count, keywords, full description, and the transcript when the video has + captions that can be retrieved; videos without captions return everything except + the transcript. A channel URL (`/channel/UC…`, `/@handle`, `/c/…`, `/user/…`) + returns its name, handle, subscriber count, video count, and full description. + When `includeImages=true`, video responses also include the thumbnail and + channel responses include the avatar. Costs the same as any other scrape. + ### Billing & errors | HTTP status | Billed? | Meaning | @@ -4256,6 +4292,7 @@ async def web_scrape_md( | 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code | | 404 | No | Target page returned or fingerprinted as not found | | 408 | No | Request timed out | + | 413 | No | Target content exceeds the maximum supported size (20 MB) | | 415 | No | Unsupported content type | | 429 | No | Per-minute rate limit exceeded; honor Retry-After | | 500 | No | Internal error | @@ -4369,6 +4406,7 @@ async def web_scrape_sitemap( domain: str, headers: Dict[str, str] | Omit = omit, max_links: int | Omit = omit, + search: str | Omit = omit, sitemap_url: str | Omit = omit, tags: SequenceNotStr[str] | Omit = omit, timeout_ms: int | Omit = omit, @@ -4381,8 +4419,13 @@ async def web_scrape_sitemap( extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> WebWebScrapeSitemapResponse: - """ - Crawl an entire website's sitemap and return all discovered page URLs. + """Crawl an entire website's sitemap and return all discovered page URLs. + + Pass + `search` to have the crawled sitemap filtered down to the pages about a phrase + (for example `pricing and plans` or `api authentication docs`), most relevant + first — a searched crawl scans the whole sitemap and costs 2 credits instead + of 1. Args: domain: Domain to build a sitemap for @@ -4394,6 +4437,10 @@ async def web_scrape_sitemap( max_links: Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Minimum is 1, maximum is 100,000. + search: Optional search phrase. When provided, the crawled sitemap is filtered to the + pages whose URLs are about that phrase, most relevant first, and the request + costs 2 credits instead of 1. + sitemap_url: Optional explicit sitemap URL. When provided, exactly this sitemap is crawled instead of discovering the domain's sitemaps. @@ -4433,6 +4480,7 @@ async def web_scrape_sitemap( "domain": domain, "headers": headers, "max_links": max_links, + "search": search, "sitemap_url": sitemap_url, "tags": tags, "timeout_ms": timeout_ms, diff --git a/src/context/dev/types/__init__.py b/src/context/dev/types/__init__.py index 11b8a4a..ca25937 100644 --- a/src/context/dev/types/__init__.py +++ b/src/context/dev/types/__init__.py @@ -17,8 +17,10 @@ from .parse_handle_params import ParseHandleParams as ParseHandleParams from .web_search_response import WebSearchResponse as WebSearchResponse from .monitor_run_response import MonitorRunResponse as MonitorRunResponse +from .person_enrich_params import PersonEnrichParams as PersonEnrichParams from .web_extract_response import WebExtractResponse as WebExtractResponse from .batch_cancel_response import BatchCancelResponse as BatchCancelResponse +from .batch_delete_response import BatchDeleteResponse as BatchDeleteResponse from .batch_submit_response import BatchSubmitResponse as BatchSubmitResponse from .brand_retrieve_params import BrandRetrieveParams as BrandRetrieveParams from .brand_search_response import BrandSearchResponse as BrandSearchResponse @@ -27,6 +29,7 @@ from .monitor_update_params import MonitorUpdateParams as MonitorUpdateParams from .parse_handle_response import ParseHandleResponse as ParseHandleResponse from .web_screenshot_params import WebScreenshotParams as WebScreenshotParams +from .person_enrich_response import PersonEnrichResponse as PersonEnrichResponse from .batch_retrieve_response import BatchRetrieveResponse as BatchRetrieveResponse from .brand_retrieve_response import BrandRetrieveResponse as BrandRetrieveResponse from .monitor_create_response import MonitorCreateResponse as MonitorCreateResponse diff --git a/src/context/dev/types/batch_delete_response.py b/src/context/dev/types/batch_delete_response.py new file mode 100644 index 0000000..49ef6b5 --- /dev/null +++ b/src/context/dev/types/batch_delete_response.py @@ -0,0 +1,35 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from .._models import BaseModel + +__all__ = ["BatchDeleteResponse", "KeyMetadata"] + + +class KeyMetadata(BaseModel): + """Metadata about the API key used for the request. + + Included in every response whenever a valid API key is provided, even when the response status is not 200. + """ + + credits_consumed: int + """The number of credits consumed by this request.""" + + credits_remaining: int + """The number of credits remaining for your organization after this request.""" + + +class BatchDeleteResponse(BaseModel): + id: Optional[str] = None + """ID of the deleted batch.""" + + deleted: Optional[bool] = None + """Always true on success.""" + + key_metadata: Optional[KeyMetadata] = None + """Metadata about the API key used for the request. + + Included in every response whenever a valid API key is provided, even when the + response status is not 200. + """ diff --git a/src/context/dev/types/batch_get_results_response.py b/src/context/dev/types/batch_get_results_response.py index 13b3ee0..dfb4d62 100644 --- a/src/context/dev/types/batch_get_results_response.py +++ b/src/context/dev/types/batch_get_results_response.py @@ -127,6 +127,13 @@ class DataOk(BaseModel): meta: Optional[Dict[str, object]] = None """Caller-supplied metadata echoed from submission.""" + ocr_pages: Optional[int] = None + """PDF pages of this document recovered by OCR (pdf.ocr=true). + + Each recovered page bills 1 credit on top of the page base credit; absent when + no OCR ran. + """ + class DataError(BaseModel): """A page the batch could not fetch.""" diff --git a/src/context/dev/types/batch_list_response.py b/src/context/dev/types/batch_list_response.py index 1267839..9a55a6e 100644 --- a/src/context/dev/types/batch_list_response.py +++ b/src/context/dev/types/batch_list_response.py @@ -25,11 +25,17 @@ class DataCredits(BaseModel): """What this batch has done to your credit balance.""" net: int - """`reserved` minus `refunded` — what the batch has cost so far. + """`reserved` minus `refunded` plus `ocr_charged` — what the batch has cost so far. Equal to `reserved` until the batch settles. """ + ocr_charged: int + """ + Credits charged for PDF pages recovered by OCR (pdf.ocr=true), 1 per recovered + page, on top of `reserved`. Stays 0 until the batch settles. + """ + refunded: int """Credits returned for pages that did not succeed. diff --git a/src/context/dev/types/batch_retrieve_response.py b/src/context/dev/types/batch_retrieve_response.py index a4cb4c6..66c6b5a 100644 --- a/src/context/dev/types/batch_retrieve_response.py +++ b/src/context/dev/types/batch_retrieve_response.py @@ -25,11 +25,17 @@ class Credits(BaseModel): """What this batch has done to your credit balance.""" net: int - """`reserved` minus `refunded` — what the batch has cost so far. + """`reserved` minus `refunded` plus `ocr_charged` — what the batch has cost so far. Equal to `reserved` until the batch settles. """ + ocr_charged: int + """ + Credits charged for PDF pages recovered by OCR (pdf.ocr=true), 1 per recovered + page, on top of `reserved`. Stays 0 until the batch settles. + """ + refunded: int """Credits returned for pages that did not succeed. diff --git a/src/context/dev/types/batch_submit_params.py b/src/context/dev/types/batch_submit_params.py index 3ae0986..d52a6d1 100644 --- a/src/context/dev/types/batch_submit_params.py +++ b/src/context/dev/types/batch_submit_params.py @@ -2,31 +2,1475 @@ from __future__ import annotations -from typing_extensions import Required, Annotated, TypedDict +from typing import Dict, Union, Iterable, Optional +from typing_extensions import Literal, Required, Annotated, TypeAlias, TypedDict from .._types import SequenceNotStr from .._utils import PropertyInfo -__all__ = ["BatchSubmitParams", "Identifiers"] +__all__ = [ + "BatchSubmitParams", + "Input", + "InputScrape", + "InputScrapeData", + "InputScrapeDataMarkdown", + "InputScrapeDataMarkdownURL", + "InputScrapeDataMarkdownOptions", + "InputScrapeDataMarkdownOptionsPdf", + "InputScrapeDataHTML", + "InputScrapeDataHtmlurL", + "InputScrapeDataHTMLOptions", + "InputScrapeDataHTMLOptionsPdf", + "InputCrawl", + "InputCrawlData", + "InputCrawlDataMarkdown", + "InputCrawlDataMarkdownSource", + "InputCrawlDataMarkdownSourceStartURL", + "InputCrawlDataMarkdownSourceStartURLControls", + "InputCrawlDataMarkdownSourceSitemap", + "InputCrawlDataMarkdownSourceSitemapControls", + "InputCrawlDataMarkdownOptions", + "InputCrawlDataMarkdownOptionsPdf", + "InputCrawlDataHTML", + "InputCrawlDataHTMLSource", + "InputCrawlDataHTMLSourceStartURL", + "InputCrawlDataHTMLSourceStartURLControls", + "InputCrawlDataHTMLSourceSitemap", + "InputCrawlDataHTMLSourceSitemapControls", + "InputCrawlDataHTMLOptions", + "InputCrawlDataHTMLOptionsPdf", +] class BatchSubmitParams(TypedDict, total=False): - identifiers: Required[Identifiers] - """Known identifiers for the person. At least one identifier is required.""" + input: Required[Input] + """Choose a URL list or a site crawl.""" tags: SequenceNotStr[str] - """Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters.""" + """Tags stored on the batch. Filter the batch list by them later.""" - timeout_ms: Annotated[int, PropertyInfo(alias="timeoutMS")] - """Optional timeout in milliseconds for the request. + webhook_url: Annotated[str, PropertyInfo(alias="webhookUrl")] + """URL notified when the batch finishes.""" - If the request takes longer than this value, it will be aborted with a 408 - status code. Maximum allowed value is 300000ms (5 minutes). + idempotency_key: Annotated[str, PropertyInfo(alias="Idempotency-Key")] + """Any string unique to this submission. + + Retries with the same key return the original batch. + """ + + +class InputScrapeDataMarkdownURL(TypedDict, total=False): + """A page to scrape, with optional data for matching results.""" + + url: Required[str] + """Page URL to scrape.""" + + item_id: Annotated[str, PropertyInfo(alias="itemId")] + """Your ID for this page, returned with its result. + + The same URL can use different IDs. + """ + + meta: Dict[str, object] + """Custom JSON returned unchanged with this page result.""" + + +class InputScrapeDataMarkdownOptionsPdf(TypedDict, total=False): + """PDF parsing controls. + + Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range. + """ + + end: int + """Last 1-based PDF page to parse. + + When omitted, parsing ends at the last page. Must be greater than or equal to + start when both are provided. + """ + + ocr: Union[bool, Literal["true", "false"]] + """ + When true, OCR the selected PDF pages that have no usable text layer (scans), + replacing each recovered page's text with the OCR result while pages with a real + text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + of the base request cost. When false, no OCR runs. + """ + + should_parse: Annotated[Union[bool, Literal["true", "false"]], PropertyInfo(alias="shouldParse")] + """When true, PDF URLs are fetched and parsed. + + When false, PDF URLs are skipped and a 400 PDF_SKIPPED is returned. + """ + + start: int + """First 1-based PDF page to parse. + + When omitted, parsing starts at the first page. + """ + + +class InputScrapeDataMarkdownOptions(TypedDict, total=False): + """Options for Markdown output.""" + + country: Literal[ + "ad", + "ae", + "af", + "ag", + "ai", + "al", + "am", + "ao", + "ar", + "at", + "au", + "aw", + "az", + "ba", + "bb", + "bd", + "be", + "bf", + "bg", + "bh", + "bi", + "bj", + "bm", + "bn", + "bo", + "bq", + "br", + "bs", + "bw", + "by", + "bz", + "ca", + "cd", + "cf", + "cg", + "ch", + "ci", + "cl", + "cm", + "cn", + "co", + "cr", + "cv", + "cw", + "cy", + "cz", + "de", + "dj", + "dk", + "dm", + "do", + "dz", + "ec", + "ee", + "eg", + "es", + "et", + "fi", + "fj", + "fr", + "ga", + "gb", + "gd", + "ge", + "gf", + "gg", + "gh", + "gm", + "gn", + "gp", + "gq", + "gr", + "gt", + "gu", + "gw", + "gy", + "hk", + "hn", + "hr", + "ht", + "hu", + "id", + "ie", + "il", + "im", + "in", + "iq", + "ir", + "is", + "it", + "je", + "jm", + "jo", + "jp", + "ke", + "kg", + "kh", + "kn", + "kr", + "kw", + "ky", + "kz", + "la", + "lb", + "lc", + "lk", + "lr", + "ls", + "lt", + "lu", + "lv", + "ly", + "ma", + "mc", + "md", + "me", + "mf", + "mg", + "mk", + "ml", + "mm", + "mn", + "mo", + "mq", + "mr", + "mt", + "mu", + "mv", + "mw", + "mx", + "my", + "mz", + "na", + "nc", + "ne", + "ng", + "ni", + "nl", + "no", + "np", + "nz", + "om", + "pa", + "pe", + "pf", + "pg", + "ph", + "pk", + "pl", + "pr", + "ps", + "pt", + "py", + "qa", + "re", + "ro", + "rs", + "ru", + "rw", + "sa", + "sc", + "sd", + "se", + "sg", + "si", + "sk", + "sl", + "sm", + "sn", + "so", + "sr", + "ss", + "st", + "sv", + "sx", + "sy", + "sz", + "tc", + "td", + "tg", + "th", + "tj", + "tl", + "tm", + "tn", + "tr", + "tt", + "tw", + "tz", + "ua", + "ug", + "us", + "uy", + "uz", + "vc", + "ve", + "vg", + "vi", + "vn", + "ye", + "yt", + "za", + "zm", + "zw", + ] + """ + Fetch the target page through a residential proxy in this country (ISO 3166-1 + alpha-2). + """ + + exclude_selectors: Annotated[Optional[SequenceNotStr[str]], PropertyInfo(alias="excludeSelectors")] + """Remove elements matching these CSS selectors. + + Applied after `includeSelectors`, so an element matching both is removed. + """ + + include_images: Annotated[bool, PropertyInfo(alias="includeImages")] + """Include image references in the Markdown.""" + + include_links: Annotated[bool, PropertyInfo(alias="includeLinks")] + """Include links in the Markdown.""" + + include_selectors: Annotated[Optional[SequenceNotStr[str]], PropertyInfo(alias="includeSelectors")] + """Keep only the subtrees matching these CSS selectors. + + Filtered pages are always fetched fresh, ignoring `maxAgeMs`. + """ + + max_age_ms: Annotated[Optional[int], PropertyInfo(alias="maxAgeMs")] + """ + Return a cached result if a prior scrape for the same parameters exists and is + younger than this many milliseconds. Defaults to 1 day (86400000 ms) when + omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. + """ + + pdf: InputScrapeDataMarkdownOptionsPdf + """PDF parsing controls. + + Use start/end to limit text extraction and embedded-image detection/OCR to an + inclusive 1-based page range. + """ + + settle_animations: Annotated[bool, PropertyInfo(alias="settleAnimations")] + """ + Wait briefly for CSS and transition animations to settle before extraction, on + pages that render in a browser. + """ + + shorten_base64_images: Annotated[bool, PropertyInfo(alias="shortenBase64Images")] + """Shorten inline base64 image data.""" + + use_main_content_only: Annotated[bool, PropertyInfo(alias="useMainContentOnly")] + """Return the main content without navigation or footers.""" + + wait_for_ms: Annotated[int, PropertyInfo(alias="waitForMs")] + """How long to wait after initial page load, in milliseconds. `0` waits 500 ms.""" + + +class InputScrapeDataMarkdown(TypedDict, total=False): + """Scrape the listed pages as Markdown.""" + + format: Required[Literal["markdown"]] + """Return page content as Markdown.""" + + urls: Required[Iterable[InputScrapeDataMarkdownURL]] + """Pages to scrape. Maximum 25000.""" + + options: InputScrapeDataMarkdownOptions + """Options for Markdown output.""" + + +class InputScrapeDataHtmlurL(TypedDict, total=False): + """A page to scrape, with optional data for matching results.""" + + url: Required[str] + """Page URL to scrape.""" + + item_id: Annotated[str, PropertyInfo(alias="itemId")] + """Your ID for this page, returned with its result. + + The same URL can use different IDs. + """ + + meta: Dict[str, object] + """Custom JSON returned unchanged with this page result.""" + + +class InputScrapeDataHTMLOptionsPdf(TypedDict, total=False): + """PDF parsing controls. + + Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range. + """ + + end: int + """Last 1-based PDF page to parse. + + When omitted, parsing ends at the last page. Must be greater than or equal to + start when both are provided. + """ + + ocr: Union[bool, Literal["true", "false"]] + """ + When true, OCR the selected PDF pages that have no usable text layer (scans), + replacing each recovered page's text with the OCR result while pages with a real + text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + of the base request cost. When false, no OCR runs. + """ + + should_parse: Annotated[Union[bool, Literal["true", "false"]], PropertyInfo(alias="shouldParse")] + """When true, PDF URLs are fetched and parsed. + + When false, PDF URLs are skipped and a 400 PDF_SKIPPED is returned. + """ + + start: int + """First 1-based PDF page to parse. + + When omitted, parsing starts at the first page. + """ + + +class InputScrapeDataHTMLOptions(TypedDict, total=False): + """Options for HTML output.""" + + country: Literal[ + "ad", + "ae", + "af", + "ag", + "ai", + "al", + "am", + "ao", + "ar", + "at", + "au", + "aw", + "az", + "ba", + "bb", + "bd", + "be", + "bf", + "bg", + "bh", + "bi", + "bj", + "bm", + "bn", + "bo", + "bq", + "br", + "bs", + "bw", + "by", + "bz", + "ca", + "cd", + "cf", + "cg", + "ch", + "ci", + "cl", + "cm", + "cn", + "co", + "cr", + "cv", + "cw", + "cy", + "cz", + "de", + "dj", + "dk", + "dm", + "do", + "dz", + "ec", + "ee", + "eg", + "es", + "et", + "fi", + "fj", + "fr", + "ga", + "gb", + "gd", + "ge", + "gf", + "gg", + "gh", + "gm", + "gn", + "gp", + "gq", + "gr", + "gt", + "gu", + "gw", + "gy", + "hk", + "hn", + "hr", + "ht", + "hu", + "id", + "ie", + "il", + "im", + "in", + "iq", + "ir", + "is", + "it", + "je", + "jm", + "jo", + "jp", + "ke", + "kg", + "kh", + "kn", + "kr", + "kw", + "ky", + "kz", + "la", + "lb", + "lc", + "lk", + "lr", + "ls", + "lt", + "lu", + "lv", + "ly", + "ma", + "mc", + "md", + "me", + "mf", + "mg", + "mk", + "ml", + "mm", + "mn", + "mo", + "mq", + "mr", + "mt", + "mu", + "mv", + "mw", + "mx", + "my", + "mz", + "na", + "nc", + "ne", + "ng", + "ni", + "nl", + "no", + "np", + "nz", + "om", + "pa", + "pe", + "pf", + "pg", + "ph", + "pk", + "pl", + "pr", + "ps", + "pt", + "py", + "qa", + "re", + "ro", + "rs", + "ru", + "rw", + "sa", + "sc", + "sd", + "se", + "sg", + "si", + "sk", + "sl", + "sm", + "sn", + "so", + "sr", + "ss", + "st", + "sv", + "sx", + "sy", + "sz", + "tc", + "td", + "tg", + "th", + "tj", + "tl", + "tm", + "tn", + "tr", + "tt", + "tw", + "tz", + "ua", + "ug", + "us", + "uy", + "uz", + "vc", + "ve", + "vg", + "vi", + "vn", + "ye", + "yt", + "za", + "zm", + "zw", + ] + """ + Fetch the target page through a residential proxy in this country (ISO 3166-1 + alpha-2). + """ + + exclude_selectors: Annotated[Optional[SequenceNotStr[str]], PropertyInfo(alias="excludeSelectors")] + """Remove elements matching these CSS selectors. + + Applied after `includeSelectors`, so an element matching both is removed. + """ + + include_selectors: Annotated[Optional[SequenceNotStr[str]], PropertyInfo(alias="includeSelectors")] + """Keep only the subtrees matching these CSS selectors. + + Filtered pages are always fetched fresh, ignoring `maxAgeMs`. + """ + + max_age_ms: Annotated[Optional[int], PropertyInfo(alias="maxAgeMs")] + """ + Return a cached result if a prior scrape for the same parameters exists and is + younger than this many milliseconds. Defaults to 1 day (86400000 ms) when + omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. + """ + + pdf: InputScrapeDataHTMLOptionsPdf + """PDF parsing controls. + + Use start/end to limit text extraction and embedded-image detection/OCR to an + inclusive 1-based page range. + """ + + settle_animations: Annotated[bool, PropertyInfo(alias="settleAnimations")] + """ + Wait briefly for CSS and transition animations to settle before extraction, on + pages that render in a browser. + """ + + use_main_content_only: Annotated[bool, PropertyInfo(alias="useMainContentOnly")] + """Return the main content without navigation or footers.""" + + wait_for_ms: Annotated[int, PropertyInfo(alias="waitForMs")] + """How long to wait after initial page load, in milliseconds. `0` waits 500 ms.""" + + +class InputScrapeDataHTML(TypedDict, total=False): + """Scrape the listed pages as HTML.""" + + format: Required[Literal["html"]] + """Return page content as HTML.""" + + urls: Required[Iterable[InputScrapeDataHtmlurL]] + """Pages to scrape. Maximum 25000.""" + + options: InputScrapeDataHTMLOptions + """Options for HTML output.""" + + +InputScrapeData: TypeAlias = Union[InputScrapeDataMarkdown, InputScrapeDataHTML] + + +class InputScrape(TypedDict, total=False): + """Scrape up to 25K URLs in one batch.""" + + data: Required[InputScrapeData] + """Pages to scrape and their output format.""" + + mode: Required[Literal["scrape"]] + """Scrape the pages in `data.urls`.""" + + +class InputCrawlDataMarkdownSourceStartURLControls(TypedDict, total=False): + """Limits and filters for page discovery.""" + + follow_subdomains: Annotated[bool, PropertyInfo(alias="followSubdomains")] + """Follow links to subdomains.""" + + max_depth: Annotated[int, PropertyInfo(alias="maxDepth")] + """Maximum link depth. Source pages are depth 0. No limit when omitted.""" + + max_urls: Annotated[int, PropertyInfo(alias="maxUrls")] + """Maximum pages to fetch. Unused reserved credits are refunded. Maximum 25000.""" + + regex: str + """RE2 pattern for URLs to include. The `start_url` itself is always included.""" + + +class InputCrawlDataMarkdownSourceStartURL(TypedDict, total=False): + """Discover pages by following links from one URL.""" + + type: Required[Literal["start_url"]] + """Start from one page.""" + + url: Required[str] + """Page where crawling begins. A URL without a scheme is read as https://.""" + + controls: InputCrawlDataMarkdownSourceStartURLControls + """Limits and filters for page discovery.""" + + +class InputCrawlDataMarkdownSourceSitemapControls(TypedDict, total=False): + """Limits and filters for the sitemap URLs. + + A sitemap batch scrapes exactly those URLs and never follows links off them, so there is no crawl depth here. + """ + + max_urls: Annotated[int, PropertyInfo(alias="maxUrls")] + """Maximum pages to fetch. Unused reserved credits are refunded. Maximum 25000.""" + + regex: str + """RE2 pattern; only sitemap URLs matching it are scraped.""" + + +class InputCrawlDataMarkdownSourceSitemap(TypedDict, total=False): + """Scrape the pages listed in a domain's sitemap. + + Links on those pages are not followed. + """ + + domain: Required[str] + """Domain whose sitemap lists the pages to scrape. + + A full URL is reduced to its domain. + """ + + type: Required[Literal["sitemap"]] + """Scrape the URLs in the domain's sitemap.""" + + controls: InputCrawlDataMarkdownSourceSitemapControls + """Limits and filters for the sitemap URLs. + + A sitemap batch scrapes exactly those URLs and never follows links off them, so + there is no crawl depth here. + """ + + +InputCrawlDataMarkdownSource: TypeAlias = Union[ + InputCrawlDataMarkdownSourceStartURL, InputCrawlDataMarkdownSourceSitemap +] + + +class InputCrawlDataMarkdownOptionsPdf(TypedDict, total=False): + """PDF parsing controls. + + Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range. + """ + + end: int + """Last 1-based PDF page to parse. + + When omitted, parsing ends at the last page. Must be greater than or equal to + start when both are provided. + """ + + ocr: Union[bool, Literal["true", "false"]] + """ + When true, OCR the selected PDF pages that have no usable text layer (scans), + replacing each recovered page's text with the OCR result while pages with a real + text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + of the base request cost. When false, no OCR runs. + """ + + should_parse: Annotated[Union[bool, Literal["true", "false"]], PropertyInfo(alias="shouldParse")] + """When true, PDF URLs are fetched and parsed. + + When false, PDF URLs are skipped and a 400 PDF_SKIPPED is returned. + """ + + start: int + """First 1-based PDF page to parse. + + When omitted, parsing starts at the first page. + """ + + +class InputCrawlDataMarkdownOptions(TypedDict, total=False): + """Options for Markdown output.""" + + country: Literal[ + "ad", + "ae", + "af", + "ag", + "ai", + "al", + "am", + "ao", + "ar", + "at", + "au", + "aw", + "az", + "ba", + "bb", + "bd", + "be", + "bf", + "bg", + "bh", + "bi", + "bj", + "bm", + "bn", + "bo", + "bq", + "br", + "bs", + "bw", + "by", + "bz", + "ca", + "cd", + "cf", + "cg", + "ch", + "ci", + "cl", + "cm", + "cn", + "co", + "cr", + "cv", + "cw", + "cy", + "cz", + "de", + "dj", + "dk", + "dm", + "do", + "dz", + "ec", + "ee", + "eg", + "es", + "et", + "fi", + "fj", + "fr", + "ga", + "gb", + "gd", + "ge", + "gf", + "gg", + "gh", + "gm", + "gn", + "gp", + "gq", + "gr", + "gt", + "gu", + "gw", + "gy", + "hk", + "hn", + "hr", + "ht", + "hu", + "id", + "ie", + "il", + "im", + "in", + "iq", + "ir", + "is", + "it", + "je", + "jm", + "jo", + "jp", + "ke", + "kg", + "kh", + "kn", + "kr", + "kw", + "ky", + "kz", + "la", + "lb", + "lc", + "lk", + "lr", + "ls", + "lt", + "lu", + "lv", + "ly", + "ma", + "mc", + "md", + "me", + "mf", + "mg", + "mk", + "ml", + "mm", + "mn", + "mo", + "mq", + "mr", + "mt", + "mu", + "mv", + "mw", + "mx", + "my", + "mz", + "na", + "nc", + "ne", + "ng", + "ni", + "nl", + "no", + "np", + "nz", + "om", + "pa", + "pe", + "pf", + "pg", + "ph", + "pk", + "pl", + "pr", + "ps", + "pt", + "py", + "qa", + "re", + "ro", + "rs", + "ru", + "rw", + "sa", + "sc", + "sd", + "se", + "sg", + "si", + "sk", + "sl", + "sm", + "sn", + "so", + "sr", + "ss", + "st", + "sv", + "sx", + "sy", + "sz", + "tc", + "td", + "tg", + "th", + "tj", + "tl", + "tm", + "tn", + "tr", + "tt", + "tw", + "tz", + "ua", + "ug", + "us", + "uy", + "uz", + "vc", + "ve", + "vg", + "vi", + "vn", + "ye", + "yt", + "za", + "zm", + "zw", + ] + """ + Fetch the target page through a residential proxy in this country (ISO 3166-1 + alpha-2). + """ + + exclude_selectors: Annotated[Optional[SequenceNotStr[str]], PropertyInfo(alias="excludeSelectors")] + """Remove elements matching these CSS selectors. + + Applied after `includeSelectors`, so an element matching both is removed. + """ + + include_images: Annotated[bool, PropertyInfo(alias="includeImages")] + """Include image references in the Markdown.""" + + include_links: Annotated[bool, PropertyInfo(alias="includeLinks")] + """Include links in the Markdown.""" + + include_selectors: Annotated[Optional[SequenceNotStr[str]], PropertyInfo(alias="includeSelectors")] + """Keep only the subtrees matching these CSS selectors. + + Filtered pages are always fetched fresh, ignoring `maxAgeMs`. + """ + + max_age_ms: Annotated[Optional[int], PropertyInfo(alias="maxAgeMs")] + """ + Return a cached result if a prior scrape for the same parameters exists and is + younger than this many milliseconds. Defaults to 1 day (86400000 ms) when + omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. + """ + + pdf: InputCrawlDataMarkdownOptionsPdf + """PDF parsing controls. + + Use start/end to limit text extraction and embedded-image detection/OCR to an + inclusive 1-based page range. + """ + + settle_animations: Annotated[bool, PropertyInfo(alias="settleAnimations")] + """ + Wait briefly for CSS and transition animations to settle before extraction, on + pages that render in a browser. + """ + + shorten_base64_images: Annotated[bool, PropertyInfo(alias="shortenBase64Images")] + """Shorten inline base64 image data.""" + + use_main_content_only: Annotated[bool, PropertyInfo(alias="useMainContentOnly")] + """Return the main content without navigation or footers.""" + + wait_for_ms: Annotated[int, PropertyInfo(alias="waitForMs")] + """How long to wait after initial page load, in milliseconds. `0` waits 500 ms.""" + + +class InputCrawlDataMarkdown(TypedDict, total=False): + """Crawl pages and return Markdown.""" + + format: Required[Literal["markdown"]] + """Return page content as Markdown.""" + + source: Required[InputCrawlDataMarkdownSource] + """How to find pages to crawl.""" + + options: InputCrawlDataMarkdownOptions + """Options for Markdown output.""" + + +class InputCrawlDataHTMLSourceStartURLControls(TypedDict, total=False): + """Limits and filters for page discovery.""" + + follow_subdomains: Annotated[bool, PropertyInfo(alias="followSubdomains")] + """Follow links to subdomains.""" + + max_depth: Annotated[int, PropertyInfo(alias="maxDepth")] + """Maximum link depth. Source pages are depth 0. No limit when omitted.""" + + max_urls: Annotated[int, PropertyInfo(alias="maxUrls")] + """Maximum pages to fetch. Unused reserved credits are refunded. Maximum 25000.""" + + regex: str + """RE2 pattern for URLs to include. The `start_url` itself is always included.""" + + +class InputCrawlDataHTMLSourceStartURL(TypedDict, total=False): + """Discover pages by following links from one URL.""" + + type: Required[Literal["start_url"]] + """Start from one page.""" + + url: Required[str] + """Page where crawling begins. A URL without a scheme is read as https://.""" + + controls: InputCrawlDataHTMLSourceStartURLControls + """Limits and filters for page discovery.""" + + +class InputCrawlDataHTMLSourceSitemapControls(TypedDict, total=False): + """Limits and filters for the sitemap URLs. + + A sitemap batch scrapes exactly those URLs and never follows links off them, so there is no crawl depth here. + """ + + max_urls: Annotated[int, PropertyInfo(alias="maxUrls")] + """Maximum pages to fetch. Unused reserved credits are refunded. Maximum 25000.""" + + regex: str + """RE2 pattern; only sitemap URLs matching it are scraped.""" + + +class InputCrawlDataHTMLSourceSitemap(TypedDict, total=False): + """Scrape the pages listed in a domain's sitemap. + + Links on those pages are not followed. + """ + + domain: Required[str] + """Domain whose sitemap lists the pages to scrape. + + A full URL is reduced to its domain. + """ + + type: Required[Literal["sitemap"]] + """Scrape the URLs in the domain's sitemap.""" + + controls: InputCrawlDataHTMLSourceSitemapControls + """Limits and filters for the sitemap URLs. + + A sitemap batch scrapes exactly those URLs and never follows links off them, so + there is no crawl depth here. + """ + + +InputCrawlDataHTMLSource: TypeAlias = Union[InputCrawlDataHTMLSourceStartURL, InputCrawlDataHTMLSourceSitemap] + + +class InputCrawlDataHTMLOptionsPdf(TypedDict, total=False): + """PDF parsing controls. + + Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range. + """ + + end: int + """Last 1-based PDF page to parse. + + When omitted, parsing ends at the last page. Must be greater than or equal to + start when both are provided. + """ + + ocr: Union[bool, Literal["true", "false"]] + """ + When true, OCR the selected PDF pages that have no usable text layer (scans), + replacing each recovered page's text with the OCR result while pages with a real + text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + of the base request cost. When false, no OCR runs. + """ + + should_parse: Annotated[Union[bool, Literal["true", "false"]], PropertyInfo(alias="shouldParse")] + """When true, PDF URLs are fetched and parsed. + + When false, PDF URLs are skipped and a 400 PDF_SKIPPED is returned. + """ + + start: int + """First 1-based PDF page to parse. + + When omitted, parsing starts at the first page. + """ + + +class InputCrawlDataHTMLOptions(TypedDict, total=False): + """Options for HTML output.""" + + country: Literal[ + "ad", + "ae", + "af", + "ag", + "ai", + "al", + "am", + "ao", + "ar", + "at", + "au", + "aw", + "az", + "ba", + "bb", + "bd", + "be", + "bf", + "bg", + "bh", + "bi", + "bj", + "bm", + "bn", + "bo", + "bq", + "br", + "bs", + "bw", + "by", + "bz", + "ca", + "cd", + "cf", + "cg", + "ch", + "ci", + "cl", + "cm", + "cn", + "co", + "cr", + "cv", + "cw", + "cy", + "cz", + "de", + "dj", + "dk", + "dm", + "do", + "dz", + "ec", + "ee", + "eg", + "es", + "et", + "fi", + "fj", + "fr", + "ga", + "gb", + "gd", + "ge", + "gf", + "gg", + "gh", + "gm", + "gn", + "gp", + "gq", + "gr", + "gt", + "gu", + "gw", + "gy", + "hk", + "hn", + "hr", + "ht", + "hu", + "id", + "ie", + "il", + "im", + "in", + "iq", + "ir", + "is", + "it", + "je", + "jm", + "jo", + "jp", + "ke", + "kg", + "kh", + "kn", + "kr", + "kw", + "ky", + "kz", + "la", + "lb", + "lc", + "lk", + "lr", + "ls", + "lt", + "lu", + "lv", + "ly", + "ma", + "mc", + "md", + "me", + "mf", + "mg", + "mk", + "ml", + "mm", + "mn", + "mo", + "mq", + "mr", + "mt", + "mu", + "mv", + "mw", + "mx", + "my", + "mz", + "na", + "nc", + "ne", + "ng", + "ni", + "nl", + "no", + "np", + "nz", + "om", + "pa", + "pe", + "pf", + "pg", + "ph", + "pk", + "pl", + "pr", + "ps", + "pt", + "py", + "qa", + "re", + "ro", + "rs", + "ru", + "rw", + "sa", + "sc", + "sd", + "se", + "sg", + "si", + "sk", + "sl", + "sm", + "sn", + "so", + "sr", + "ss", + "st", + "sv", + "sx", + "sy", + "sz", + "tc", + "td", + "tg", + "th", + "tj", + "tl", + "tm", + "tn", + "tr", + "tt", + "tw", + "tz", + "ua", + "ug", + "us", + "uy", + "uz", + "vc", + "ve", + "vg", + "vi", + "vn", + "ye", + "yt", + "za", + "zm", + "zw", + ] + """ + Fetch the target page through a residential proxy in this country (ISO 3166-1 + alpha-2). + """ + + exclude_selectors: Annotated[Optional[SequenceNotStr[str]], PropertyInfo(alias="excludeSelectors")] + """Remove elements matching these CSS selectors. + + Applied after `includeSelectors`, so an element matching both is removed. """ + include_selectors: Annotated[Optional[SequenceNotStr[str]], PropertyInfo(alias="includeSelectors")] + """Keep only the subtrees matching these CSS selectors. + + Filtered pages are always fetched fresh, ignoring `maxAgeMs`. + """ + + max_age_ms: Annotated[Optional[int], PropertyInfo(alias="maxAgeMs")] + """ + Return a cached result if a prior scrape for the same parameters exists and is + younger than this many milliseconds. Defaults to 1 day (86400000 ms) when + omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. + """ + + pdf: InputCrawlDataHTMLOptionsPdf + """PDF parsing controls. + + Use start/end to limit text extraction and embedded-image detection/OCR to an + inclusive 1-based page range. + """ + + settle_animations: Annotated[bool, PropertyInfo(alias="settleAnimations")] + """ + Wait briefly for CSS and transition animations to settle before extraction, on + pages that render in a browser. + """ + + use_main_content_only: Annotated[bool, PropertyInfo(alias="useMainContentOnly")] + """Return the main content without navigation or footers.""" + + wait_for_ms: Annotated[int, PropertyInfo(alias="waitForMs")] + """How long to wait after initial page load, in milliseconds. `0` waits 500 ms.""" + + +class InputCrawlDataHTML(TypedDict, total=False): + """Crawl pages and return HTML.""" + + format: Required[Literal["html"]] + """Return page content as HTML.""" + + source: Required[InputCrawlDataHTMLSource] + """How to find pages to crawl.""" + + options: InputCrawlDataHTMLOptions + """Options for HTML output.""" + + +InputCrawlData: TypeAlias = Union[InputCrawlDataMarkdown, InputCrawlDataHTML] + + +class InputCrawl(TypedDict, total=False): + """Crawl pages starting from a URL or from a domain's sitemap.""" + + data: Required[InputCrawlData] + """Crawl source and output format.""" + + mode: Required[Literal["crawl"]] + """Discover and scrape pages from `data.source`.""" -class Identifiers(TypedDict, total=False): - """Known identifiers for the person. At least one identifier is required.""" - linkedin_url: Annotated[str, PropertyInfo(alias="linkedinUrl")] - """LinkedIn profile URL, e.g. https://www.linkedin.com/in/yahia-bakour/.""" +Input: TypeAlias = Union[InputScrape, InputCrawl] diff --git a/src/context/dev/types/batch_submit_response.py b/src/context/dev/types/batch_submit_response.py index 6fd2690..89be81d 100644 --- a/src/context/dev/types/batch_submit_response.py +++ b/src/context/dev/types/batch_submit_response.py @@ -3,241 +3,33 @@ from typing import List, Optional from typing_extensions import Literal -from pydantic import Field as FieldInfo - +from .intake import Intake from .._models import BaseModel +from .crawl_controls import CrawlControls -__all__ = [ - "BatchSubmitResponse", - "Metadata", - "MetadataIdentifiers", - "Person", - "PersonEducation", - "PersonEducationInstitution", - "PersonEducationDates", - "PersonEducationDatesEndDate", - "PersonEducationDatesStartDate", - "PersonExperience", - "PersonExperienceCompany", - "PersonExperienceDates", - "PersonExperienceDatesEndDate", - "PersonExperienceDatesStartDate", - "PersonProfile", - "PersonSkill", - "KeyMetadata", -] - - -class MetadataIdentifiers(BaseModel): - """Identifiers returned for the person.""" - - linkedin_url: Optional[str] = FieldInfo(alias="linkedinUrl", default=None) - """LinkedIn profile URL.""" - - -class Metadata(BaseModel): - """Additional response details.""" - - identifiers: MetadataIdentifiers - """Identifiers returned for the person.""" - - sources_attempted: List[Literal["linkedin", "cv", "manual", "github", "other"]] = FieldInfo( - alias="sourcesAttempted" - ) - """Source categories checked.""" - - sources_succeeded: List[Literal["linkedin", "cv", "manual", "github", "other"]] = FieldInfo( - alias="sourcesSucceeded" - ) - """Source categories with data.""" - - urls_analyzed: List[str] = FieldInfo(alias="urlsAnalyzed") - """URLs reviewed for this profile.""" - - personal_website_url: Optional[str] = FieldInfo(alias="personalWebsiteUrl", default=None) - """Personal website URL, when found.""" - - -class PersonEducationInstitution(BaseModel): - """School or institution name.""" - - display: str - """Display name.""" - - normalized: Optional[str] = None - """Standardized name, when available.""" - - -class PersonEducationDatesEndDate(BaseModel): - """End date, when known.""" - - year: int - """Year value.""" - - day: Optional[int] = None - """Day value, when known.""" - - month: Optional[int] = None - """Month value, when known.""" - - -class PersonEducationDatesStartDate(BaseModel): - """Start date, when known.""" - - year: int - """Year value.""" - - day: Optional[int] = None - """Day value, when known.""" - - month: Optional[int] = None - """Month value, when known.""" - - -class PersonEducationDates(BaseModel): - """Education dates.""" - - end_date: Optional[PersonEducationDatesEndDate] = FieldInfo(alias="endDate", default=None) - """End date, when known.""" - - is_current: Optional[bool] = FieldInfo(alias="isCurrent", default=None) - """Whether the entry is current.""" - - start_date: Optional[PersonEducationDatesStartDate] = FieldInfo(alias="startDate", default=None) - """Start date, when known.""" - - -class PersonEducation(BaseModel): - institution: PersonEducationInstitution - """School or institution name.""" - - dates: Optional[PersonEducationDates] = None - """Education dates.""" - - description: Optional[str] = None - """Additional education details.""" - - field_of_study: Optional[str] = FieldInfo(alias="fieldOfStudy", default=None) - """Area of study.""" - - qualification: Optional[str] = None - """Degree, certificate, or credential.""" - - -class PersonExperienceCompany(BaseModel): - """Company or organization name.""" - - display: str - """Display name.""" - - normalized: Optional[str] = None - """Standardized name, when available.""" - +__all__ = ["BatchSubmitResponse", "Credits", "InvalidURL", "KeyMetadata"] -class PersonExperienceDatesEndDate(BaseModel): - """End date, when known.""" - year: int - """Year value.""" +class Credits(BaseModel): + """What accepting this batch cost.""" - day: Optional[int] = None - """Day value, when known.""" + reserved: int + """Credits just debited from your balance. - month: Optional[int] = None - """Month value, when known.""" - - -class PersonExperienceDatesStartDate(BaseModel): - """Start date, when known.""" - - year: int - """Year value.""" - - day: Optional[int] = None - """Day value, when known.""" - - month: Optional[int] = None - """Month value, when known.""" - - -class PersonExperienceDates(BaseModel): - """Role dates.""" - - end_date: Optional[PersonExperienceDatesEndDate] = FieldInfo(alias="endDate", default=None) - """End date, when known.""" - - is_current: Optional[bool] = FieldInfo(alias="isCurrent", default=None) - """Whether the entry is current.""" - - start_date: Optional[PersonExperienceDatesStartDate] = FieldInfo(alias="startDate", default=None) - """Start date, when known.""" - - -class PersonExperience(BaseModel): - company: PersonExperienceCompany - """Company or organization name.""" - - title: str - """Role or job title.""" - - dates: Optional[PersonExperienceDates] = None - """Role dates.""" - - description: Optional[str] = None - """Role description.""" - - -class PersonProfile(BaseModel): - """Core profile details.""" - - full_name: Optional[str] = FieldInfo(alias="fullName", default=None) - """Person's full name.""" - - headline: Optional[str] = None - """Short professional headline.""" - - location: Optional[str] = None - """Person's listed location.""" - - profile_picture_url: Optional[str] = FieldInfo(alias="profilePictureUrl", default=None) - """Profile image URL.""" - - summary: Optional[str] = None - """Brief profile summary.""" - - -class PersonSkill(BaseModel): - name: str - """Skill name.""" - - normalized: Optional[str] = None - """Standardized skill name, when available.""" - - proficiency: Optional[str] = None - """Skill proficiency, when available.""" - - -class Person(BaseModel): - """Retrieved person profile.""" - - education: List[PersonEducation] - """Education history.""" + Whatever the batch does not spend is refunded when it settles. + """ - experience: List[PersonExperience] - """Work history.""" - profile: PersonProfile - """Core profile details.""" +class InvalidURL(BaseModel): + reason: str + """Why it was rejected.""" - skills: List[PersonSkill] - """Listed skills.""" + url: str + """Rejected URL.""" class KeyMetadata(BaseModel): - """Metadata about the API key used for the request. - - Included in every response whenever a valid API key is provided, even when the response status is not 200. - """ + """API key usage for this request.""" credits_consumed: int """The number of credits consumed by this request.""" @@ -247,21 +39,44 @@ class KeyMetadata(BaseModel): class BatchSubmitResponse(BaseModel): - code: Literal[200] - """HTTP status code.""" + id: str + """Batch ID. Poll GET /batch/{batch_id} with it.""" + + crawl: Optional[CrawlControls] = None + """ + The crawl controls as submitted, so the limits requested can be compared against + what the crawl reached. + """ + + created_at: str + """When the batch was created.""" - metadata: Metadata - """Additional response details.""" + credits: Credits + """What accepting this batch cost.""" - person: Person - """Retrieved person profile.""" + format: Literal["markdown", "html"] + """What each page will be returned as.""" - status: Literal["ok"] - """Response status.""" + input: Intake + """What submission took in, and what it charged for.""" + + invalid_urls: List[InvalidURL] + """Rejected URLs, up to 100. These are not charged.""" + + mode: Literal["scrape", "crawl"] + """How pages will be selected.""" + + status: Literal["queued"] + """Always `queued`. An accepted batch has not started yet.""" + + tags: List[str] + """Tags stored on the batch.""" key_metadata: Optional[KeyMetadata] = None - """Metadata about the API key used for the request. + """API key usage for this request.""" + + webhook_secret: Optional[str] = None + """Signing secret for the completion webhook, returned only here and never again. - Included in every response whenever a valid API key is provided, even when the - response status is not 200. + Store it now; it is not repeated by GET /batch/{batch_id}. """ diff --git a/src/context/dev/types/brand_retrieve_response.py b/src/context/dev/types/brand_retrieve_response.py index 51fdb59..e732559 100644 --- a/src/context/dev/types/brand_retrieve_response.py +++ b/src/context/dev/types/brand_retrieve_response.py @@ -13,6 +13,7 @@ "BrandBackdropColor", "BrandBackdropResolution", "BrandColor", + "BrandEmployees", "BrandIndustries", "BrandIndustriesEic", "BrandLinks", @@ -90,6 +91,20 @@ class BrandColor(BaseModel): """Name of the color""" +class BrandEmployees(BaseModel): + """Employee headcount information for the brand (will be null if unknown)""" + + exact: Optional[int] = None + """Exact employee count when a precise headcount is known""" + + range: Optional[ + Literal[ + "1 to 10", "11 to 50", "51 to 200", "201 to 500", "501 to 1000", "1001 to 5000", "5001 to 10000", "10001+" + ] + ] = None + """Employee count range for the brand (e.g. '11 to 50')""" + + class BrandIndustriesEic(BaseModel): industry: Literal[ "Aerospace & Defense", @@ -490,6 +505,9 @@ class Brand(BaseModel): email: Optional[str] = None """Company email address""" + employees: Optional[BrandEmployees] = None + """Employee headcount information for the brand (will be null if unknown)""" + industries: Optional[BrandIndustries] = None """Industry classification information for the brand""" diff --git a/src/context/dev/types/brand_search_response.py b/src/context/dev/types/brand_search_response.py index f2b46ed..8bd1ed9 100644 --- a/src/context/dev/types/brand_search_response.py +++ b/src/context/dev/types/brand_search_response.py @@ -36,7 +36,10 @@ class KeyMetadata(BaseModel): class BrandSearchResponse(BaseModel): results: List[Result] - """Up to 10 matching brands, most popular first. Empty when nothing matches.""" + """ + Up to 10 matching brands, name matches first, then domain matches, most popular + first within each group. Empty when nothing matches. + """ key_metadata: Optional[KeyMetadata] = None """Metadata about the API key used for the request. diff --git a/src/context/dev/types/crawl_controls.py b/src/context/dev/types/crawl_controls.py index 182a460..4ed12c1 100644 --- a/src/context/dev/types/crawl_controls.py +++ b/src/context/dev/types/crawl_controls.py @@ -5,24 +5,28 @@ from .._models import BaseModel -__all__ = ["CrawlControls", "Source", "SourceUnionMember0", "SourceUnionMember1"] +__all__ = ["CrawlControls", "Source", "SourceStartURL", "SourceSitemap"] -class SourceUnionMember0(BaseModel): +class SourceStartURL(BaseModel): + """The crawl discovered pages by following links from one URL.""" + type: Literal["start_url"] url: str """Page the crawl started from.""" -class SourceUnionMember1(BaseModel): +class SourceSitemap(BaseModel): + """The crawl scraped the pages listed in the domain's sitemap.""" + domain: str """Domain whose sitemap supplied the pages.""" type: Literal["sitemap"] -Source: TypeAlias = Union[SourceUnionMember0, SourceUnionMember1] +Source: TypeAlias = Union[SourceStartURL, SourceSitemap] class CrawlControls(BaseModel): diff --git a/src/context/dev/types/parse_handle_params.py b/src/context/dev/types/parse_handle_params.py index e48e164..26676e1 100644 --- a/src/context/dev/types/parse_handle_params.py +++ b/src/context/dev/types/parse_handle_params.py @@ -100,10 +100,11 @@ class ParseHandleParams(TypedDict, total=False): ocr: Union[bool, Literal["true", "false"]] """ - When true for PDF inputs, detect and OCR images embedded in the selected pages, - inserting recognized text at each image's position in page reading order while - preserving the PDF text layer. pdf.start/pdf.end limit the inclusive page range. - When false, all OCR is disabled, including the automatic scanned-PDF fallback. + When true for PDF inputs, OCR the selected pages that have no usable text layer + (scans), replacing each recovered page's text with the OCR result while pages + with a real text layer keep it. pdf.start/pdf.end limit the inclusive page + range. Billed at 1 credit per page OCR actually recovered, on top of the base + request cost. When false, no OCR runs. """ pdf: Pdf diff --git a/src/context/dev/types/person_enrich_params.py b/src/context/dev/types/person_enrich_params.py new file mode 100644 index 0000000..feb1255 --- /dev/null +++ b/src/context/dev/types/person_enrich_params.py @@ -0,0 +1,71 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Iterable +from typing_extensions import Annotated, TypedDict + +from .._types import SequenceNotStr +from .._utils import PropertyInfo + +__all__ = ["PersonEnrichParams", "Company", "Education", "EducationInstitution", "Location", "Name"] + + +class PersonEnrichParams(TypedDict, total=False): + company: Company + + education: Iterable[Education] + + email: str + + location: Location + + name: Name + + social_urls: SequenceNotStr[str] + + tags: SequenceNotStr[str] + """Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters.""" + + timeout_ms: Annotated[int, PropertyInfo(alias="timeoutMS")] + """Optional timeout in milliseconds for the request. + + If the request takes longer than this value, it will be aborted with a 408 + status code. Maximum allowed value is 300000ms (5 minutes). + """ + + +class Company(TypedDict, total=False): + domain: str + + name: str + + +class EducationInstitution(TypedDict, total=False): + domain: str + + name: str + + +class Education(TypedDict, total=False): + degree: str + + field_of_study: str + + graduation_year: int + + institution: EducationInstitution + + +class Location(TypedDict, total=False): + city: str + + country: str + + region: str + + +class Name(TypedDict, total=False): + first: str + + last: str diff --git a/src/context/dev/types/person_enrich_response.py b/src/context/dev/types/person_enrich_response.py new file mode 100644 index 0000000..34cc571 --- /dev/null +++ b/src/context/dev/types/person_enrich_response.py @@ -0,0 +1,236 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Union, Optional +from typing_extensions import Literal, Annotated, TypeAlias + +from .._utils import PropertyInfo +from .._models import BaseModel + +__all__ = [ + "PersonEnrichResponse", + "Match", + "MatchPersonEnrichmentCandidateMatch", + "MatchPersonEnrichmentCandidateMatchPerson", + "MatchPersonEnrichmentCandidateMatchPersonEducation", + "MatchPersonEnrichmentCandidateMatchPersonEducationInstitution", + "MatchPersonEnrichmentCandidateMatchPersonEducationEndDate", + "MatchPersonEnrichmentCandidateMatchPersonEducationStartDate", + "MatchPersonEnrichmentCandidateMatchPersonExperience", + "MatchPersonEnrichmentCandidateMatchPersonExperienceOrganization", + "MatchPersonEnrichmentCandidateMatchPersonExperienceEndDate", + "MatchPersonEnrichmentCandidateMatchPersonExperienceStartDate", + "MatchPersonEnrichmentCandidateMatchPersonCurrentRole", + "MatchPersonEnrichmentCandidateMatchPersonCurrentRoleOrganization", + "MatchPersonEnrichmentCandidateMatchPersonCurrentRoleEndDate", + "MatchPersonEnrichmentCandidateMatchPersonCurrentRoleStartDate", + "MatchPersonEnrichmentCandidateMatchPersonLocation", + "MatchPersonEnrichmentCandidateMatchPersonName", + "MatchPersonEnrichmentNotFoundMatch", + "KeyMetadata", +] + + +class MatchPersonEnrichmentCandidateMatchPersonEducationInstitution(BaseModel): + name: str + + domain: Optional[str] = None + + +class MatchPersonEnrichmentCandidateMatchPersonEducationEndDate(BaseModel): + year: int + + day: Optional[int] = None + + month: Optional[int] = None + + +class MatchPersonEnrichmentCandidateMatchPersonEducationStartDate(BaseModel): + year: int + + day: Optional[int] = None + + month: Optional[int] = None + + +class MatchPersonEnrichmentCandidateMatchPersonEducation(BaseModel): + institution: MatchPersonEnrichmentCandidateMatchPersonEducationInstitution + + degree: Optional[str] = None + + description: Optional[str] = None + + end_date: Optional[MatchPersonEnrichmentCandidateMatchPersonEducationEndDate] = None + + field_of_study: Optional[str] = None + + start_date: Optional[MatchPersonEnrichmentCandidateMatchPersonEducationStartDate] = None + + +class MatchPersonEnrichmentCandidateMatchPersonExperienceOrganization(BaseModel): + name: str + + domain: Optional[str] = None + + +class MatchPersonEnrichmentCandidateMatchPersonExperienceEndDate(BaseModel): + year: int + + day: Optional[int] = None + + month: Optional[int] = None + + +class MatchPersonEnrichmentCandidateMatchPersonExperienceStartDate(BaseModel): + year: int + + day: Optional[int] = None + + month: Optional[int] = None + + +class MatchPersonEnrichmentCandidateMatchPersonExperience(BaseModel): + organization: MatchPersonEnrichmentCandidateMatchPersonExperienceOrganization + + title: str + + description: Optional[str] = None + + end_date: Optional[MatchPersonEnrichmentCandidateMatchPersonExperienceEndDate] = None + + is_current: Optional[bool] = None + + location: Optional[str] = None + + start_date: Optional[MatchPersonEnrichmentCandidateMatchPersonExperienceStartDate] = None + + +class MatchPersonEnrichmentCandidateMatchPersonCurrentRoleOrganization(BaseModel): + name: str + + domain: Optional[str] = None + + +class MatchPersonEnrichmentCandidateMatchPersonCurrentRoleEndDate(BaseModel): + year: int + + day: Optional[int] = None + + month: Optional[int] = None + + +class MatchPersonEnrichmentCandidateMatchPersonCurrentRoleStartDate(BaseModel): + year: int + + day: Optional[int] = None + + month: Optional[int] = None + + +class MatchPersonEnrichmentCandidateMatchPersonCurrentRole(BaseModel): + organization: MatchPersonEnrichmentCandidateMatchPersonCurrentRoleOrganization + + title: str + + description: Optional[str] = None + + end_date: Optional[MatchPersonEnrichmentCandidateMatchPersonCurrentRoleEndDate] = None + + is_current: Optional[bool] = None + + location: Optional[str] = None + + start_date: Optional[MatchPersonEnrichmentCandidateMatchPersonCurrentRoleStartDate] = None + + +class MatchPersonEnrichmentCandidateMatchPersonLocation(BaseModel): + city: Optional[str] = None + + country: Optional[str] = None + + country_code: Optional[str] = None + + display: Optional[str] = None + + region: Optional[str] = None + + +class MatchPersonEnrichmentCandidateMatchPersonName(BaseModel): + first: Optional[str] = None + + full: Optional[str] = None + + last: Optional[str] = None + + +class MatchPersonEnrichmentCandidateMatchPerson(BaseModel): + education: List[MatchPersonEnrichmentCandidateMatchPersonEducation] + + experience: List[MatchPersonEnrichmentCandidateMatchPersonExperience] + + skills: List[str] + + social_urls: List[str] + + website_urls: List[str] + + avatar_url: Optional[str] = None + + bio: Optional[str] = None + + current_role: Optional[MatchPersonEnrichmentCandidateMatchPersonCurrentRole] = None + + email: Optional[str] = None + + location: Optional[MatchPersonEnrichmentCandidateMatchPersonLocation] = None + + name: Optional[MatchPersonEnrichmentCandidateMatchPersonName] = None + + +class MatchPersonEnrichmentCandidateMatch(BaseModel): + """The highest-scoring person candidate.""" + + person: MatchPersonEnrichmentCandidateMatchPerson + + score: int + + status: Literal["candidate"] + + +class MatchPersonEnrichmentNotFoundMatch(BaseModel): + """No usable person candidate was found.""" + + person: None = None + + score: None = None + + status: Literal["not_found"] + + +Match: TypeAlias = Annotated[ + Union[MatchPersonEnrichmentCandidateMatch, MatchPersonEnrichmentNotFoundMatch], PropertyInfo(discriminator="status") +] + + +class KeyMetadata(BaseModel): + """Metadata about the API key used for the request. + + Included in every response whenever a valid API key is provided, even when the response status is not 200. + """ + + credits_consumed: int + """The number of credits consumed by this request.""" + + credits_remaining: int + """The number of credits remaining for your organization after this request.""" + + +class PersonEnrichResponse(BaseModel): + match: Match + """The highest-scoring person candidate.""" + + key_metadata: Optional[KeyMetadata] = None + """Metadata about the API key used for the request. + + Included in every response whenever a valid API key is provided, even when the + response status is not 200. + """ diff --git a/src/context/dev/types/web_search_response.py b/src/context/dev/types/web_search_response.py index 62d46ed..3636c95 100644 --- a/src/context/dev/types/web_search_response.py +++ b/src/context/dev/types/web_search_response.py @@ -11,7 +11,7 @@ class ResultMarkdown(BaseModel): """Markdown scrape status and content for this result.""" - code: Literal["SUCCESS", "NOT_REQUESTED", "TIMEOUT", "WEBSITE_ACCESS_ERROR", "ERROR"] + code: Literal["SUCCESS", "NOT_REQUESTED", "TIMEOUT", "CONTENT_TOO_LARGE", "WEBSITE_ACCESS_ERROR", "ERROR"] """Per-result scrape outcome. Inspect this before reading `markdown`.""" markdown: Optional[str] = None diff --git a/src/context/dev/types/web_web_crawl_md_params.py b/src/context/dev/types/web_web_crawl_md_params.py index aa455b3..9bd49a6 100644 --- a/src/context/dev/types/web_web_crawl_md_params.py +++ b/src/context/dev/types/web_web_crawl_md_params.py @@ -346,9 +346,10 @@ class Pdf(TypedDict, total=False): ocr: bool """ - When true, detect and OCR images embedded in the selected PDF pages, inserting - recognized text at each image's position in page reading order while preserving - the PDF text layer. This is separate from automatic scanned-PDF OCR fallback. + When true, OCR the selected PDF pages that have no usable text layer (scans), + replacing each recovered page's text with the OCR result while pages with a real + text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + of the base request cost. """ should_parse: Annotated[bool, PropertyInfo(alias="shouldParse")] diff --git a/src/context/dev/types/web_web_scrape_html_params.py b/src/context/dev/types/web_web_scrape_html_params.py index 4ddbb15..461974a 100644 --- a/src/context/dev/types/web_web_scrape_html_params.py +++ b/src/context/dev/types/web_web_scrape_html_params.py @@ -348,15 +348,16 @@ class Pdf(TypedDict, total=False): ocr: Union[bool, Literal["true", "false"]] """ - When true, detect and OCR images embedded in the selected PDF pages, inserting - recognized text at each image's position in page reading order while preserving - the PDF text layer. This is separate from automatic scanned-PDF OCR fallback. + When true, OCR the selected PDF pages that have no usable text layer (scans), + replacing each recovered page's text with the OCR result while pages with a real + text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + of the base request cost. When false, no OCR runs. """ should_parse: Annotated[Union[bool, Literal["true", "false"]], PropertyInfo(alias="shouldParse")] """When true, PDF URLs are fetched and parsed. - When false, PDF URLs are skipped and a 400 WEBSITE_ACCESS_ERROR is returned. + When false, PDF URLs are skipped and a 400 PDF_SKIPPED is returned. """ start: int diff --git a/src/context/dev/types/web_web_scrape_md_params.py b/src/context/dev/types/web_web_scrape_md_params.py index 7cc894e..11b56f1 100644 --- a/src/context/dev/types/web_web_scrape_md_params.py +++ b/src/context/dev/types/web_web_scrape_md_params.py @@ -360,15 +360,16 @@ class Pdf(TypedDict, total=False): ocr: Union[bool, Literal["true", "false"]] """ - When true, detect and OCR images embedded in the selected PDF pages, inserting - recognized text at each image's position in page reading order while preserving - the PDF text layer. This is separate from automatic scanned-PDF OCR fallback. + When true, OCR the selected PDF pages that have no usable text layer (scans), + replacing each recovered page's text with the OCR result while pages with a real + text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + of the base request cost. When false, no OCR runs. """ should_parse: Annotated[Union[bool, Literal["true", "false"]], PropertyInfo(alias="shouldParse")] """When true, PDF URLs are fetched and parsed. - When false, PDF URLs are skipped and a 400 WEBSITE_ACCESS_ERROR is returned. + When false, PDF URLs are skipped and a 400 PDF_SKIPPED is returned. """ start: int diff --git a/src/context/dev/types/web_web_scrape_sitemap_params.py b/src/context/dev/types/web_web_scrape_sitemap_params.py index 468747d..b14c7d2 100644 --- a/src/context/dev/types/web_web_scrape_sitemap_params.py +++ b/src/context/dev/types/web_web_scrape_sitemap_params.py @@ -28,6 +28,13 @@ class WebWebScrapeSitemapParams(TypedDict, total=False): Defaults to 10,000. Minimum is 1, maximum is 100,000. """ + search: str + """Optional search phrase. + + When provided, the crawled sitemap is filtered to the pages whose URLs are about + that phrase, most relevant first, and the request costs 2 credits instead of 1. + """ + sitemap_url: Annotated[str, PropertyInfo(alias="sitemapUrl")] """Optional explicit sitemap URL. diff --git a/src/context/dev/types/web_web_scrape_sitemap_response.py b/src/context/dev/types/web_web_scrape_sitemap_response.py index 331a0e0..73d479b 100644 --- a/src/context/dev/types/web_web_scrape_sitemap_response.py +++ b/src/context/dev/types/web_web_scrape_sitemap_response.py @@ -50,7 +50,10 @@ class WebWebScrapeSitemapResponse(BaseModel): """Indicates success""" urls: List[str] - """Array of discovered page URLs from the sitemap (max 500)""" + """Discovered page URLs from the sitemap, up to `maxLinks`. + + When `search` is set these are only the matching pages, most relevant first. + """ key_metadata: Optional[KeyMetadata] = None """Metadata about the API key used for the request. diff --git a/tests/api_resources/test_batch.py b/tests/api_resources/test_batch.py index 2cce1cf..0691496 100644 --- a/tests/api_resources/test_batch.py +++ b/tests/api_resources/test_batch.py @@ -12,6 +12,7 @@ from context.dev.types import ( BatchListResponse, BatchCancelResponse, + BatchDeleteResponse, BatchSubmitResponse, BatchRetrieveResponse, BatchGetResultsResponse, @@ -106,6 +107,48 @@ def test_streaming_response_list(self, client: ContextDev) -> None: assert cast(Any, response.is_closed) is True + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_delete(self, client: ContextDev) -> None: + batch = client.batch.delete( + "batch_9f2c8a", + ) + assert_matches_type(BatchDeleteResponse, batch, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_delete(self, client: ContextDev) -> None: + response = client.batch.with_raw_response.delete( + "batch_9f2c8a", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + batch = response.parse() + assert_matches_type(BatchDeleteResponse, batch, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_delete(self, client: ContextDev) -> None: + with client.batch.with_streaming_response.delete( + "batch_9f2c8a", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + batch = response.parse() + assert_matches_type(BatchDeleteResponse, batch, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_delete(self, client: ContextDev) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `batch_id` but received ''"): + client.batch.with_raw_response.delete( + "", + ) + @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_cancel(self, client: ContextDev) -> None: @@ -204,7 +247,16 @@ def test_path_params_get_results(self, client: ContextDev) -> None: @parametrize def test_method_submit(self, client: ContextDev) -> None: batch = client.batch.submit( - identifiers={}, + input={ + "data": { + "format": "markdown", + "urls": [ + {"url": "https://example.com/products/anvil"}, + {"url": "https://example.com/products/hammer"}, + ], + }, + "mode": "scrape", + }, ) assert_matches_type(BatchSubmitResponse, batch, path=["response"]) @@ -212,9 +264,45 @@ def test_method_submit(self, client: ContextDev) -> None: @parametrize def test_method_submit_with_all_params(self, client: ContextDev) -> None: batch = client.batch.submit( - identifiers={"linkedin_url": "https://www.linkedin.com/in/yahia-bakour/"}, - tags=["production", "team-alpha"], - timeout_ms=1000, + input={ + "data": { + "format": "markdown", + "urls": [ + { + "url": "https://example.com/products/anvil", + "item_id": "sku-1", + "meta": {"category": "bar"}, + }, + { + "url": "https://example.com/products/hammer", + "item_id": "sku-2", + "meta": {"foo": "bar"}, + }, + ], + "options": { + "country": "de", + "exclude_selectors": ["x"], + "include_images": True, + "include_links": True, + "include_selectors": ["x"], + "max_age_ms": 0, + "pdf": { + "end": 1, + "ocr": "true", + "should_parse": "true", + "start": 1, + }, + "settle_animations": True, + "shorten_base64_images": True, + "use_main_content_only": True, + "wait_for_ms": 0, + }, + }, + "mode": "scrape", + }, + tags=["docs", "competitor"], + webhook_url="webhookUrl", + idempotency_key="Idempotency-Key", ) assert_matches_type(BatchSubmitResponse, batch, path=["response"]) @@ -222,7 +310,16 @@ def test_method_submit_with_all_params(self, client: ContextDev) -> None: @parametrize def test_raw_response_submit(self, client: ContextDev) -> None: response = client.batch.with_raw_response.submit( - identifiers={}, + input={ + "data": { + "format": "markdown", + "urls": [ + {"url": "https://example.com/products/anvil"}, + {"url": "https://example.com/products/hammer"}, + ], + }, + "mode": "scrape", + }, ) assert response.is_closed is True @@ -234,7 +331,16 @@ def test_raw_response_submit(self, client: ContextDev) -> None: @parametrize def test_streaming_response_submit(self, client: ContextDev) -> None: with client.batch.with_streaming_response.submit( - identifiers={}, + input={ + "data": { + "format": "markdown", + "urls": [ + {"url": "https://example.com/products/anvil"}, + {"url": "https://example.com/products/hammer"}, + ], + }, + "mode": "scrape", + }, ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" @@ -333,6 +439,48 @@ async def test_streaming_response_list(self, async_client: AsyncContextDev) -> N assert cast(Any, response.is_closed) is True + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_delete(self, async_client: AsyncContextDev) -> None: + batch = await async_client.batch.delete( + "batch_9f2c8a", + ) + assert_matches_type(BatchDeleteResponse, batch, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_delete(self, async_client: AsyncContextDev) -> None: + response = await async_client.batch.with_raw_response.delete( + "batch_9f2c8a", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + batch = await response.parse() + assert_matches_type(BatchDeleteResponse, batch, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_delete(self, async_client: AsyncContextDev) -> None: + async with async_client.batch.with_streaming_response.delete( + "batch_9f2c8a", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + batch = await response.parse() + assert_matches_type(BatchDeleteResponse, batch, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_delete(self, async_client: AsyncContextDev) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `batch_id` but received ''"): + await async_client.batch.with_raw_response.delete( + "", + ) + @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_cancel(self, async_client: AsyncContextDev) -> None: @@ -431,7 +579,16 @@ async def test_path_params_get_results(self, async_client: AsyncContextDev) -> N @parametrize async def test_method_submit(self, async_client: AsyncContextDev) -> None: batch = await async_client.batch.submit( - identifiers={}, + input={ + "data": { + "format": "markdown", + "urls": [ + {"url": "https://example.com/products/anvil"}, + {"url": "https://example.com/products/hammer"}, + ], + }, + "mode": "scrape", + }, ) assert_matches_type(BatchSubmitResponse, batch, path=["response"]) @@ -439,9 +596,45 @@ async def test_method_submit(self, async_client: AsyncContextDev) -> None: @parametrize async def test_method_submit_with_all_params(self, async_client: AsyncContextDev) -> None: batch = await async_client.batch.submit( - identifiers={"linkedin_url": "https://www.linkedin.com/in/yahia-bakour/"}, - tags=["production", "team-alpha"], - timeout_ms=1000, + input={ + "data": { + "format": "markdown", + "urls": [ + { + "url": "https://example.com/products/anvil", + "item_id": "sku-1", + "meta": {"category": "bar"}, + }, + { + "url": "https://example.com/products/hammer", + "item_id": "sku-2", + "meta": {"foo": "bar"}, + }, + ], + "options": { + "country": "de", + "exclude_selectors": ["x"], + "include_images": True, + "include_links": True, + "include_selectors": ["x"], + "max_age_ms": 0, + "pdf": { + "end": 1, + "ocr": "true", + "should_parse": "true", + "start": 1, + }, + "settle_animations": True, + "shorten_base64_images": True, + "use_main_content_only": True, + "wait_for_ms": 0, + }, + }, + "mode": "scrape", + }, + tags=["docs", "competitor"], + webhook_url="webhookUrl", + idempotency_key="Idempotency-Key", ) assert_matches_type(BatchSubmitResponse, batch, path=["response"]) @@ -449,7 +642,16 @@ async def test_method_submit_with_all_params(self, async_client: AsyncContextDev @parametrize async def test_raw_response_submit(self, async_client: AsyncContextDev) -> None: response = await async_client.batch.with_raw_response.submit( - identifiers={}, + input={ + "data": { + "format": "markdown", + "urls": [ + {"url": "https://example.com/products/anvil"}, + {"url": "https://example.com/products/hammer"}, + ], + }, + "mode": "scrape", + }, ) assert response.is_closed is True @@ -461,7 +663,16 @@ async def test_raw_response_submit(self, async_client: AsyncContextDev) -> None: @parametrize async def test_streaming_response_submit(self, async_client: AsyncContextDev) -> None: async with async_client.batch.with_streaming_response.submit( - identifiers={}, + input={ + "data": { + "format": "markdown", + "urls": [ + {"url": "https://example.com/products/anvil"}, + {"url": "https://example.com/products/hammer"}, + ], + }, + "mode": "scrape", + }, ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" diff --git a/tests/api_resources/test_people.py b/tests/api_resources/test_people.py new file mode 100644 index 0000000..4d3801a --- /dev/null +++ b/tests/api_resources/test_people.py @@ -0,0 +1,150 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import os +from typing import Any, cast + +import pytest + +from context.dev import ContextDev, AsyncContextDev +from tests.utils import assert_matches_type +from context.dev.types import PersonEnrichResponse + +base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") + + +class TestPeople: + parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_enrich(self, client: ContextDev) -> None: + person = client.people.enrich() + assert_matches_type(PersonEnrichResponse, person, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_enrich_with_all_params(self, client: ContextDev) -> None: + person = client.people.enrich( + company={ + "domain": "analyticalengines.example", + "name": "Analytical Engines", + }, + education=[ + { + "degree": "x", + "field_of_study": "x", + "graduation_year": 1900, + "institution": { + "domain": "x", + "name": "x", + }, + } + ], + email="dev@stainless.com", + location={ + "city": "x", + "country": "x", + "region": "x", + }, + name={ + "first": "Ada", + "last": "Lovelace", + }, + social_urls=["https://www.linkedin.com/in/ada-lovelace/"], + tags=["production", "team-alpha"], + timeout_ms=1000, + ) + assert_matches_type(PersonEnrichResponse, person, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_enrich(self, client: ContextDev) -> None: + response = client.people.with_raw_response.enrich() + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + person = response.parse() + assert_matches_type(PersonEnrichResponse, person, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_enrich(self, client: ContextDev) -> None: + with client.people.with_streaming_response.enrich() as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + person = response.parse() + assert_matches_type(PersonEnrichResponse, person, path=["response"]) + + assert cast(Any, response.is_closed) is True + + +class TestAsyncPeople: + parametrize = pytest.mark.parametrize( + "async_client", [False, True, {"http_client": "aiohttp"}], indirect=True, ids=["loose", "strict", "aiohttp"] + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_enrich(self, async_client: AsyncContextDev) -> None: + person = await async_client.people.enrich() + assert_matches_type(PersonEnrichResponse, person, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_enrich_with_all_params(self, async_client: AsyncContextDev) -> None: + person = await async_client.people.enrich( + company={ + "domain": "analyticalengines.example", + "name": "Analytical Engines", + }, + education=[ + { + "degree": "x", + "field_of_study": "x", + "graduation_year": 1900, + "institution": { + "domain": "x", + "name": "x", + }, + } + ], + email="dev@stainless.com", + location={ + "city": "x", + "country": "x", + "region": "x", + }, + name={ + "first": "Ada", + "last": "Lovelace", + }, + social_urls=["https://www.linkedin.com/in/ada-lovelace/"], + tags=["production", "team-alpha"], + timeout_ms=1000, + ) + assert_matches_type(PersonEnrichResponse, person, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_enrich(self, async_client: AsyncContextDev) -> None: + response = await async_client.people.with_raw_response.enrich() + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + person = await response.parse() + assert_matches_type(PersonEnrichResponse, person, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_enrich(self, async_client: AsyncContextDev) -> None: + async with async_client.people.with_streaming_response.enrich() as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + person = await response.parse() + assert_matches_type(PersonEnrichResponse, person, path=["response"]) + + assert cast(Any, response.is_closed) is True diff --git a/tests/api_resources/test_web.py b/tests/api_resources/test_web.py index 95c09b9..b873aab 100644 --- a/tests/api_resources/test_web.py +++ b/tests/api_resources/test_web.py @@ -633,6 +633,7 @@ def test_method_web_scrape_sitemap_with_all_params(self, client: ContextDev) -> domain="xxx", headers={"foo": "J!"}, max_links=1, + search="help center and troubleshooting articles", sitemap_url="https://example.com", tags=["production", "team-alpha"], timeout_ms=1, @@ -1277,6 +1278,7 @@ async def test_method_web_scrape_sitemap_with_all_params(self, async_client: Asy domain="xxx", headers={"foo": "J!"}, max_links=1, + search="help center and troubleshooting articles", sitemap_url="https://example.com", tags=["production", "team-alpha"], timeout_ms=1,