harvester.endpoint package

Submodules

harvester.endpoint.oads module

class harvester.endpoint.oads.IndexFileParser(use_oads_ext: bool)

Bases: object

apply_assets(record: Dict[str, str], item: Item)

Create and add STAC Assets from the provided information

apply_eo_extension(record: Dict[str, str], item: Item)

Apply properties from the EO extension spec

apply_oads_extension(record: Dict[str, str], item: Item)

Apply properties from the non-standard OADS extension spec

apply_properties(record: Dict[str, str], item: Item)

Apply any optional properties currently in the core spec

apply_sar_extension(record: Dict[str, str], item: Item)

Apply properties from the SAR extension spec

apply_sat_extension(record: Dict[str, str], item: Item)

Apply properties from the SAT extension spec

apply_version_extension(record: Dict[str, str], item: Item)

Apply properties from the Version extension spec

apply_view_extension(record: Dict[str, str], item: Item)

Apply properties from the View extension spec

parse(record: Dict[str, str]) → Item

Parses a record to a STAC Item

parse_bbox(record: Dict[str, str]) → List[float] | None

Parses the bbox from the potential “boundingBox” field.

parse_geometry(record: Dict[str, str]) → dict | None

Retrieve GeoJSON geometry from records “footprint”, “nominalTrack” or “sceneCentre” fields (in that preference)

class harvester.endpoint.oads.OADSEndpoint(url: str, use_oads_ext: bool = False)

Bases: Endpoint

harvest() → Iterator[dict]

Harvests an OADS system. First performs a “Scan” operation to get the list of index files which are subsequently retrieved and parsed.

harvest_index_file(index_file_name: str) → Iterator[dict]

Harvests a single index file and yields the contents as STAC Items

type = 'OADS'

harvester.endpoint.oads.camel_to_snake_case(value: str) → str

harvester.endpoint.oads.create_oads_endpoint(resource_config: ResourceConfig) → Endpoint

harvester.endpoint.oads.pairwise(iterable)

s -> (s0, s1), (s2, s3), (s4, s5), …

harvester.endpoint.oads.pairwise_iterative(iterable: List[List[float]]) → Iterator[Tuple[List[float], List[float]]]

s -> (s0,s1), (s1,s2), (s2, s3), …

harvester.endpoint.oads.parse_coord_list(raw: str) → List[List[float]]

harvester.endpoint.oads.tripletwise(iterable)

s -> (s0, s1, s2), (s3, s4, s5), (s6, s7, s8), …

harvester.endpoint.opensearch module

class harvester.endpoint.opensearch.AtomFormat

Bases: OpenSearchFormat

mimetype: str = 'application/atom+xml'

parse()

Parser of different opensearch formats

Parameters:

response (requests.Response) – response fetched from server

Returns:

a single search page

Return type:

SearchPage

class harvester.endpoint.opensearch.GeoJSONFormat(property_mapping: Dict[str, str])

Bases: OpenSearchFormat

mimetype: str = 'application/json'

parse(response: Response) → SearchPage

Parser of different opensearch formats

Parameters:

response (requests.Response) – response fetched from server

Returns:

a single search page

Return type:

SearchPage

class harvester.endpoint.opensearch.OpenSearchEndpoint(url: str, format_config: OpenSearchFormat, query: OpenSearchQuery)

Bases: Endpoint

NS = {'': 'http://www.w3.org/2005/Atom', 'cql': 'http://a9.com/-/opensearch/extensions/cql/1.0/', 'dc': 'http://purl.org/dc/elements/1.1/', 'eo': 'http://a9.com/-/opensearch/extensions/eo/1.0/', 'geo': 'http://a9.com/-/opensearch/extensions/geo/1.0/', 'georss': 'http://www.georss.org/georss/', 'media': 'http://search.yahoo.com/mrss/', 'opensearch': 'http://a9.com/-/spec/opensearch/1.1/', 'owc': 'http://www.opengis.net/owc/1.0/', 'parameters': 'http://a9.com/-/spec/opensearch/extensions/parameters/1.0/', 'time': 'http://a9.com/-/opensearch/extensions/time/1.0/'}

harvest() → Iterator[dict]

Starts the harvesting of the resource, returning an iterator of the harvested items.

type = 'OpenSearch'

class harvester.endpoint.opensearch.OpenSearchFormat

Bases: ABC

mimetype: str

abstract parse(response: Response) → SearchPage

Parser of different opensearch formats

Parameters:

response (requests.Response) – response fetched from server

Returns:

a single search page

Return type:

SearchPage

class harvester.endpoint.opensearch.OpenSearchQuery(*args, **kwargs)

Bases: Query

prepare_params(*args, **kwargs)

class harvester.endpoint.opensearch.SearchPage(index: int, total: int, records: List[dict] = <factory>)

Bases: object

index: int

records: List[dict]

total: int

harvester.endpoint.opensearch.create_atom_format(format_config: FormatConfig) → OpenSearchFormat

harvester.endpoint.opensearch.create_geojson_format(format_config: FormatConfig) → OpenSearchFormat

harvester.endpoint.opensearch.create_opensearch_endpoint(resource_config: ResourceConfig) → Endpoint

harvester.endpoint.opensearch.create_opensearch_format(format_config: FormatConfig) → OpenSearchFormat

harvester.endpoint.opensearch.create_opensearch_query(query_config: QueryConfig) → OpenSearchQuery

harvester.endpoint.query module

class harvester.endpoint.query.Query(time: TimeConfig = TimeConfig(begin='2023-05-10T11:32:48.827412', end='2023-05-10T11:32:48.827420'), bbox: str | None = None, collection: str | None = None)

Bases: ABC

abstract prepare_params(*args, **kwargs)

harvester.endpoint.request module

class harvester.endpoint.request.RetrySession(total: int = 5, backoff_factor: float = 0.2, status_forcelist: List[int] = [429, 500, 502, 503, 504], **kwargs: dict | None)

Bases: Session

A class extending requests.Session to add retry functionality with exponential backoff

harvester.endpoint.stacapi module

class harvester.endpoint.stacapi.STACAPIEndpoint(url: str, query: StacAPIQuery)

Bases: Endpoint

harvest() → Iterator[dict]

Starts the harvesting of the resource, returning an iterator of the harvested items.

type = 'STACAPI'

class harvester.endpoint.stacapi.StacAPIQuery(*args, **kwargs)

Bases: Query

prepare_params(*args, **kwargs) → dict

harvester.endpoint.stacapi.create_stacapi_endpoint(resource_config: ResourceConfig) → Endpoint

harvester.endpoint.stacapi.create_stacapi_query(query_config: QueryConfig) → StacAPIQuery

Module contents

class harvester.endpoint.OADSEndpoint(url: str, use_oads_ext: bool = False)

Bases: Endpoint

harvest() → Iterator[dict]

Harvests an OADS system. First performs a “Scan” operation to get the list of index files which are subsequently retrieved and parsed.

harvest_index_file(index_file_name: str) → Iterator[dict]

Harvests a single index file and yields the contents as STAC Items

type = 'OADS'

class harvester.endpoint.OpenSearchEndpoint(url: str, format_config: OpenSearchFormat, query: OpenSearchQuery)

Bases: Endpoint

NS = {'': 'http://www.w3.org/2005/Atom', 'cql': 'http://a9.com/-/opensearch/extensions/cql/1.0/', 'dc': 'http://purl.org/dc/elements/1.1/', 'eo': 'http://a9.com/-/opensearch/extensions/eo/1.0/', 'geo': 'http://a9.com/-/opensearch/extensions/geo/1.0/', 'georss': 'http://www.georss.org/georss/', 'media': 'http://search.yahoo.com/mrss/', 'opensearch': 'http://a9.com/-/spec/opensearch/1.1/', 'owc': 'http://www.opengis.net/owc/1.0/', 'parameters': 'http://a9.com/-/spec/opensearch/extensions/parameters/1.0/', 'time': 'http://a9.com/-/opensearch/extensions/time/1.0/'}

harvest() → Iterator[dict]

Starts the harvesting of the resource, returning an iterator of the harvested items.

type = 'OpenSearch'

class harvester.endpoint.STACAPIEndpoint(url: str, query: StacAPIQuery)

Bases: Endpoint

harvest() → Iterator[dict]

Starts the harvesting of the resource, returning an iterator of the harvested items.

type = 'STACAPI'

harvester.endpoint.create_oads_endpoint(resource_config: ResourceConfig) → Endpoint

harvester.endpoint.create_opensearch_endpoint(resource_config: ResourceConfig) → Endpoint

harvester.endpoint.create_stacapi_endpoint(resource_config: ResourceConfig) → Endpoint

harvester.endpoint.get_endpoint(resource_config: ResourceConfig) → Endpoint | None

Retrieves endpoint from mapping if found

Parameters:

resource_config (ResourceConfig) – Resource configuration

Returns:

Initialized endpoint from ENDPOINT_MAP

Return type:

Optional[Endpoint]