Skip to content

Commit d775354

Browse files
committed
[skip ci] squash and rebaase initial persistence boiler plate
1 parent 1738cdc commit d775354

28 files changed

Lines changed: 5909 additions & 0 deletions
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# SCM syntax highlighting & preventing 3-way merges
2+
pixi.lock merge=binary linguist-language=YAML linguist-generated=true -diff
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# pixi environments
2+
.pixi/*
3+
!.pixi/config.toml
4+
report.xml
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Persistence Model for use with the PyEarthTools Package
2+
3+
**TODO: description**
4+
5+
## Installation
6+
7+
Clone the repository, then run
8+
```shell
9+
pip install -e .
10+
```
11+
12+
## Training
13+
14+
No training is required for this model. It computes persistence on-the-fly using historical data loaded via the PET pipeline.
15+
16+
## Predictions / Inference
17+
18+
You can generate persistence values out of the box using the `pet predict` command line API, or by using a Jupyter Notebook as demonstrated in the tutorial gallery.
19+
20+
```shell
21+
pet predict
22+
```
23+
24+
and `Development/Persistence` should be visible.
25+
26+
If so, you can now run some inference.
27+
28+
```shell
29+
pet predict --model Development/Persistence
30+
```
31+
32+
When running the command, it will prompt for other required arguments.
33+
34+
**TODO: description of required arguments**
35+
36+
37+
#### Example
38+
39+
```shell
40+
pet predict --model Development/Persistence # TODO
41+
```
42+
43+
## Acknowledgments
44+
45+
Not applicable. Heuristically developed.

packages/bundled_models/persistence/pixi.lock

Lines changed: 3369 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
[build-system]
2+
requires = ["setuptools"]
3+
build-backend = "setuptools.build_meta"
4+
5+
[project]
6+
name = "pyearthtools-persistence"
7+
version = "0.6.0"
8+
description = "Persistence Bundled Model"
9+
readme = "README.md"
10+
requires-python = ">=3.11, <3.14"
11+
keywords = ["persistence", "pyearthtools", "models"]
12+
maintainers = [
13+
{name = "Tennessee Leeuwenburg", email = "tennessee.leeuwenburg@bom.gov.au"},
14+
{name = "Nikeeth Ramanathan", email = "nikeeth.ramanathan@gmail.com"},
15+
]
16+
classifiers = [
17+
"License :: OSI Approved :: Apache Software License",
18+
"Operating System :: OS Independent",
19+
"Programming Language :: Python :: 3.11",
20+
"Programming Language :: Python :: 3.12",
21+
"Programming Language :: Python :: 3.13",
22+
]
23+
dependencies = [
24+
'pyearthtools.zoo>=0.5.0',
25+
'pyearthtools.data>=0.5.0',
26+
'pyearthtools.pipeline>=0.5.0',
27+
'hydra-core',
28+
]
29+
[dependency-groups]
30+
dev = [
31+
"pytest>=8.4.2",
32+
"ruff",
33+
"pytest-cov",
34+
"pytest-xdist",
35+
]
36+
37+
[project.urls]
38+
homepage = "https://pyearthtools.readthedocs.io/"
39+
documentation = "https://pyearthtools.readthedocs.io/"
40+
repository = "https://github.com/ACCESS-Community-Hub/PyEarthTools"
41+
42+
[project.entry-points."pyearthtools.zoo.model"]
43+
Global_PERSIST = "persistence.registered_model:Persistence"
44+
45+
[tool.isort]
46+
profile = "black"
47+
48+
[tool.black]
49+
line-length = 120
50+
51+
[tool.mypy]
52+
warn_return_any = true
53+
warn_unused_configs = true
54+
55+
[[tool.mypy.overrides]]
56+
ignore_missing_imports = true
57+
58+
[tool.hatch.version]
59+
path = "src/persistence/__init__.py"
60+
61+
[tool.hatch.build.targets.wheel]
62+
packages = ["src/pyearthtools/"]
63+
64+
[tool.pixi.workspace]
65+
channels = ["conda-forge"]
66+
platforms = ["linux-64"]
67+
68+
[tool.pixi.pypi-dependencies]
69+
pyearthtools-persistence = { path = ".", editable = true }
70+
71+
[tool.pixi.tasks]
72+
73+
[tool.pixi.dependencies]
74+
python = ">=3.11,<3.14"
75+
xarray = ">=2026.1.0,<2027"
76+
77+
[tool.pixi.feature.testing.dependencies]
78+
pytest = ">=9.0.2,<10"
79+
pytest-cov = ">=7.0.0,<8"
80+
pytest-xdist = ">=3.8.0,<4"
81+
ruff = ">=0.15.0,<0.16"
82+
ipython = ">=9.10.0,<10"
83+
84+
[tool.pixi.feature.dask.dependencies]
85+
dask-core = "*"
86+
distributed = "*"
87+
pyarrow = ">=23.0.0,<24"
88+
89+
[tool.pixi.environments]
90+
dask = ["dask"]
91+
dev = ["dask", "testing"]

packages/bundled_models/persistence/src/persistence/__init__.py

Whitespace-only changes.
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
from contextlib import contextmanager
2+
3+
4+
# default scheduler string to set "single-threaded" mode.
5+
_STR_DASK_SYNC_SCHEDULER = "synchronous"
6+
7+
8+
@contextmanager
9+
def _set_synchronous_dask():
10+
"""
11+
Wrapper to set `dask` to single-threaded mode. Note: "single-threaded" in `dask`-land
12+
(specifically) is the same as "synchronous".
13+
14+
This handles the case where dask is _not_ installed. In which case it does a pass-through.
15+
16+
IMPORTANT: never nest this context manager or call dask.config.reset() or attempt to update any
17+
configs inside this context. Doing so may invalidate the "synchronous" setting.
18+
19+
Example:
20+
def do_stuff(...):
21+
# I can now (optionally) fork other processes here - without confusing dask.
22+
# IMPORTANT: I shouldn't try to reintroduce parallelism using dask here
23+
...
24+
25+
with _set_synchronous_dask():
26+
do_stuff(...)
27+
"""
28+
try:
29+
# this import order is important for the "distributed" configs to be recognized
30+
import dask
31+
import dask.config
32+
33+
# NOTE: if you don't have dask.distributed, this setting may not work as intended.
34+
# so you will have to manually deal with it in the compute level.
35+
import dask.distributed
36+
37+
# set state to desired config
38+
with dask.config.set(scheduler=_STR_DASK_SYNC_SCHEDULER):
39+
yield
40+
except ImportError:
41+
yield

packages/bundled_models/persistence/src/persistence/interface/__init__.py

Whitespace-only changes.
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
from enum import StrEnum, auto
2+
3+
4+
class PersistenceBackendType(StrEnum):
5+
"""
6+
Enumeration of supported compute backends for persistence computations.
7+
8+
---
9+
10+
SUPPORTED BACKENDS (as of 2026-02-28):
11+
- NUMPY (20260228)
12+
- others are WIP
13+
14+
Note: "supported" implies that the backend is supported by the build system, it does not imply
15+
that the particular persistence method itself is supported for that backend.
16+
17+
---
18+
19+
Backends are configured at the "build" level in pyproject.toml, e.g. for rust this may be
20+
maturin/pyO3, which usually handles most of the heavy lifting.
21+
22+
numba might require certain system dependencies - e.g. llvm, to function since it requires
23+
building on the fly.
24+
25+
For C/zig this would involve using:
26+
a. ziglang/zig-pypi to build the zig packages into wheels and running them on the fly using
27+
sys.execute to execute the wheel as a module, building/running zig on-the-fly. Avoids
28+
having to distribute the pre-built dependencies, but may not work well with specific
29+
interfaces like `numpy`.
30+
b. using setuptools-zig to build them into a "integrated" library and packaging the build
31+
into the wheel/distribution
32+
c. using cffi or ctypes.
33+
34+
Methods a. and b. would require extending Python.h directly, and hence are preferrable, since
35+
they don't involve foreign calls. Unlike numba, method a. exists for zig where jit compilation
36+
can happen without dependency on additional system libraries.
37+
38+
All of the above methods generally avoid (or at least have the ability to avoid) the need for
39+
conda environments and are pretty light weight.
40+
"""
41+
42+
C = "c"
43+
C_ZIG = "zig"
44+
NUMBA = "numba"
45+
NUMPY = "numpy"
46+
RUST = "rust"
47+
UNKNOWN = auto()
48+
49+
def check_support(self):
50+
"""
51+
As per the module documentation, this method only tells you if a particular backend is
52+
supported by the *build system*, it doesn't imply that the backend is useable for any given
53+
method.
54+
55+
Therefore, this check can and should be done as early as possible. Whereas method
56+
compatiblilty will be checked later into the runtime but still early enough point in the
57+
code, before attempting the computation. (see `PersistenceCompute` for more details)
58+
"""
59+
match self:
60+
case PersistenceBackendType.NUMPY:
61+
return
62+
case _:
63+
raise NotImplementedError(
64+
f"PersistenceBackendType: {self} is not supported"
65+
)

0 commit comments

Comments
 (0)