Skip to content

Commit 5d3587b

Browse files
committed
Add comprehensive tests for ApiSpec and related classes
- Introduced unit tests for enums: AuthType, OAuthFlow, HTTPMethod, AggregationType, RuleAction, BackoffType, and ResponseFormat. - Implemented tests for Request, Pagination, Records, Processor, Rule, Iterate, Response, Call, Endpoint, and DynamicEndpoint classes. - Added tests for ApiSpec to validate programmatic building, serialization, and parsing of YAML files. - Included validation tests to ensure correct structure and constraints of ApiSpec. - Enhanced error handling for parsing functions.
1 parent 7139c6b commit 5d3587b

5 files changed

Lines changed: 2547 additions & 7 deletions

File tree

README.md

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,144 @@ pipeline.run()
468468
```
469469

470470

471+
### Building API Specs with `ApiSpec`
472+
473+
Build [API Spec](https://docs.slingdata.io/concepts/api-specs) YAML files programmatically with type checking and validation. API specs define how Sling extracts data from REST APIs.
474+
475+
```python
476+
from sling.api_spec import (
477+
ApiSpec, Endpoint, Request, Pagination, Response, Records,
478+
Processor, Rule, Iterate, Call, DynamicEndpoint,
479+
AuthType, HTTPMethod, RuleAction, AggregationType, BackoffType, ResponseFormat,
480+
)
481+
482+
spec = ApiSpec(
483+
name="My API",
484+
description="Extract data from My API",
485+
queues=["user_ids"],
486+
487+
defaults=Endpoint(
488+
state={"base_url": "https://api.example.com/v1", "limit": 100},
489+
request=Request(
490+
headers={
491+
"Authorization": 'Bearer {require(secrets.api_key, "api_key required")}',
492+
"Accept": "application/json",
493+
},
494+
rate=5,
495+
concurrency=3,
496+
),
497+
response=Response(
498+
records=Records(jmespath="data[]", primary_key=["id"]),
499+
rules=[
500+
Rule(
501+
action=RuleAction.RETRY,
502+
condition="response.status == 429",
503+
max_attempts=5,
504+
backoff=BackoffType.EXPONENTIAL,
505+
backoff_base=2,
506+
),
507+
],
508+
),
509+
pagination=Pagination(
510+
next_state={"offset": "{state.offset + state.limit}"},
511+
stop_condition="length(response.records) < state.limit",
512+
),
513+
),
514+
515+
endpoints={
516+
"users": Endpoint(
517+
description="List all users",
518+
state={"offset": 0},
519+
request=Request(
520+
url="{state.base_url}/users",
521+
parameters={"limit": "{state.limit}", "offset": "{state.offset}"},
522+
),
523+
response=Response(
524+
processors=[
525+
Processor(expression="record.id", output="queue.user_ids"),
526+
],
527+
),
528+
),
529+
530+
"user_orders": Endpoint(
531+
description="Get orders for each user",
532+
iterate=Iterate(over="queue.user_ids", into="state.user_id", concurrency=5),
533+
request=Request(url="{state.base_url}/users/{state.user_id}/orders"),
534+
response=Response(
535+
processors=[
536+
Processor(expression="state.user_id", output="record.user_id"),
537+
],
538+
),
539+
),
540+
541+
"metrics": Endpoint(
542+
description="Daily metrics (incremental)",
543+
state={
544+
"offset": 0,
545+
"since": '{coalesce(sync.last_date, date_format(date_add(now(), -30, "day"), "%Y-%m-%d"))}',
546+
},
547+
sync=["last_date"],
548+
request=Request(
549+
url="{state.base_url}/metrics",
550+
parameters={"since": "{state.since}"},
551+
),
552+
response=Response(
553+
records=Records(primary_key=["id"], update_key="date"),
554+
processors=[
555+
Processor(
556+
expression="record.date",
557+
output="state.last_date",
558+
aggregation=AggregationType.MAXIMUM,
559+
),
560+
],
561+
),
562+
),
563+
},
564+
)
565+
566+
# Validate
567+
errors = spec.validate()
568+
assert errors == [], errors
569+
570+
# Write to file
571+
spec.to_yaml_file("my_api.yaml")
572+
573+
# Or get as string
574+
print(spec.to_yaml())
575+
print(spec.to_json())
576+
```
577+
578+
Parse an existing spec:
579+
580+
```python
581+
from sling.api_spec import ApiSpec, Endpoint, Request, Response, Records
582+
583+
spec = ApiSpec.parse_file("path/to/spec.yaml")
584+
print(spec.name)
585+
print(list(spec.endpoints.keys()))
586+
587+
# Modify and re-export
588+
spec.endpoints["new_endpoint"] = Endpoint(
589+
request=Request(url="{state.base_url}/new"),
590+
response=Response(records=Records(primary_key=["id"])),
591+
)
592+
spec.to_yaml_file("updated_spec.yaml")
593+
```
594+
595+
Use `+rules`/`+processors` modifiers to append to defaults without replacing them:
596+
597+
```python
598+
from sling.api_spec import Endpoint, Request, Response, Rule, RuleAction
599+
600+
endpoint = Endpoint(
601+
request=Request(url="{state.base_url}/fragile"),
602+
response=Response(
603+
# append_rules serializes as "rules+" in YAML, keeping default rules intact
604+
append_rules=[Rule(action=RuleAction.SKIP, condition="response.status == 404")],
605+
),
606+
)
607+
```
608+
471609
## Testing
472610

473611
```bash

sling/sling/__init__.py

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,10 @@
3838

3939
class JsonEncoder(JSONEncoder):
4040
def default(self, o):
41+
from enum import Enum as BaseEnum
4142
if hasattr(o, 'to_dict'):
4243
return o.to_dict()
43-
elif isinstance(o, Mode):
44-
return o.value
45-
elif isinstance(o, Format):
46-
return o.value
47-
elif isinstance(o, Compression):
48-
return o.value
49-
elif isinstance(o, MergeStrategy):
44+
elif isinstance(o, BaseEnum):
5045
return o.value
5146
elif isinstance(o, datetime.datetime):
5247
return str(o)

0 commit comments

Comments
 (0)