diff --git a/cmd/api/api/snapshots.go b/cmd/api/api/snapshots.go index da5cd9e0..620d9574 100644 --- a/cmd/api/api/snapshots.go +++ b/cmd/api/api/snapshots.go @@ -3,6 +3,7 @@ package api import ( "context" "errors" + "time" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" @@ -203,6 +204,124 @@ func (s *ApiService) ForkSnapshot(ctx context.Context, request oapi.ForkSnapshot return oapi.ForkSnapshot201JSONResponse(instanceToOAPI(*result)), nil } +// GetInstanceSnapshotSchedule gets a snapshot schedule for the resolved instance. +func (s *ApiService) GetInstanceSnapshotSchedule(ctx context.Context, request oapi.GetInstanceSnapshotScheduleRequestObject) (oapi.GetInstanceSnapshotScheduleResponseObject, error) { + inst := mw.GetResolvedInstance[instances.Instance](ctx) + if inst == nil { + return oapi.GetInstanceSnapshotSchedule500JSONResponse{Code: "internal_error", Message: "resource not resolved"}, nil + } + + scheduleManager, ok := s.InstanceManager.(instances.SnapshotScheduleManager) + if !ok { + return oapi.GetInstanceSnapshotSchedule500JSONResponse{Code: "internal_error", Message: "snapshot scheduling is not available"}, nil + } + + schedule, err := scheduleManager.GetSnapshotSchedule(ctx, inst.Id) + if err != nil { + log := logger.FromContext(ctx) + switch { + case errors.Is(err, instances.ErrSnapshotScheduleNotFound): + return oapi.GetInstanceSnapshotSchedule404JSONResponse{Code: "not_found", Message: "snapshot schedule not found"}, nil + case errors.Is(err, instances.ErrNotFound): + return oapi.GetInstanceSnapshotSchedule404JSONResponse{Code: "not_found", Message: "instance not found"}, nil + default: + log.ErrorContext(ctx, "failed to get snapshot schedule", "error", err) + return oapi.GetInstanceSnapshotSchedule500JSONResponse{Code: "internal_error", Message: "failed to get snapshot schedule"}, nil + } + } + + return oapi.GetInstanceSnapshotSchedule200JSONResponse(snapshotScheduleToOAPI(*schedule)), nil +} + +// SetInstanceSnapshotSchedule creates or updates a snapshot schedule for the resolved instance. +func (s *ApiService) SetInstanceSnapshotSchedule(ctx context.Context, request oapi.SetInstanceSnapshotScheduleRequestObject) (oapi.SetInstanceSnapshotScheduleResponseObject, error) { + inst := mw.GetResolvedInstance[instances.Instance](ctx) + if inst == nil { + return oapi.SetInstanceSnapshotSchedule500JSONResponse{Code: "internal_error", Message: "resource not resolved"}, nil + } + if request.Body == nil { + return oapi.SetInstanceSnapshotSchedule400JSONResponse{Code: "invalid_request", Message: "request body is required"}, nil + } + + scheduleManager, ok := s.InstanceManager.(instances.SnapshotScheduleManager) + if !ok { + return oapi.SetInstanceSnapshotSchedule500JSONResponse{Code: "internal_error", Message: "snapshot scheduling is not available"}, nil + } + + interval, err := time.ParseDuration(request.Body.Interval) + if err != nil { + return oapi.SetInstanceSnapshotSchedule400JSONResponse{Code: "invalid_request", Message: "interval must be a valid duration"}, nil + } + if request.Body.Retention.MaxCount == nil && request.Body.Retention.MaxAge == nil { + return oapi.SetInstanceSnapshotSchedule400JSONResponse{Code: "invalid_request", Message: "retention must include max_count or max_age"}, nil + } + + retention := instances.SnapshotScheduleRetention{} + if request.Body.Retention.MaxCount != nil { + retention.MaxCount = *request.Body.Retention.MaxCount + } + if request.Body.Retention.MaxAge != nil { + maxAge, parseErr := time.ParseDuration(*request.Body.Retention.MaxAge) + if parseErr != nil { + return oapi.SetInstanceSnapshotSchedule400JSONResponse{Code: "invalid_request", Message: "retention.max_age must be a valid duration"}, nil + } + retention.MaxAge = maxAge + } + req := instances.SetSnapshotScheduleRequest{ + Interval: interval, + Metadata: toMapTags(request.Body.Metadata), + Retention: retention, + } + if request.Body.NamePrefix != nil { + req.NamePrefix = *request.Body.NamePrefix + } + + schedule, err := scheduleManager.SetSnapshotSchedule(ctx, inst.Id, req) + if err != nil { + log := logger.FromContext(ctx) + switch { + case errors.Is(err, instances.ErrInvalidRequest): + return oapi.SetInstanceSnapshotSchedule400JSONResponse{Code: "invalid_request", Message: err.Error()}, nil + case errors.Is(err, instances.ErrNotFound): + return oapi.SetInstanceSnapshotSchedule404JSONResponse{Code: "not_found", Message: "instance not found"}, nil + default: + log.ErrorContext(ctx, "failed to set snapshot schedule", "error", err) + return oapi.SetInstanceSnapshotSchedule500JSONResponse{Code: "internal_error", Message: "failed to set snapshot schedule"}, nil + } + } + + return oapi.SetInstanceSnapshotSchedule200JSONResponse(snapshotScheduleToOAPI(*schedule)), nil +} + +// DeleteInstanceSnapshotSchedule deletes a snapshot schedule for the resolved instance. +func (s *ApiService) DeleteInstanceSnapshotSchedule(ctx context.Context, request oapi.DeleteInstanceSnapshotScheduleRequestObject) (oapi.DeleteInstanceSnapshotScheduleResponseObject, error) { + inst := mw.GetResolvedInstance[instances.Instance](ctx) + if inst == nil { + return oapi.DeleteInstanceSnapshotSchedule500JSONResponse{Code: "internal_error", Message: "resource not resolved"}, nil + } + + scheduleManager, ok := s.InstanceManager.(instances.SnapshotScheduleManager) + if !ok { + return oapi.DeleteInstanceSnapshotSchedule500JSONResponse{Code: "internal_error", Message: "snapshot scheduling is not available"}, nil + } + + err := scheduleManager.DeleteSnapshotSchedule(ctx, inst.Id) + if err != nil { + log := logger.FromContext(ctx) + switch { + case errors.Is(err, instances.ErrSnapshotScheduleNotFound): + return oapi.DeleteInstanceSnapshotSchedule404JSONResponse{Code: "not_found", Message: "snapshot schedule not found"}, nil + case errors.Is(err, instances.ErrNotFound): + return oapi.DeleteInstanceSnapshotSchedule404JSONResponse{Code: "not_found", Message: "instance not found"}, nil + default: + log.ErrorContext(ctx, "failed to delete snapshot schedule", "error", err) + return oapi.DeleteInstanceSnapshotSchedule500JSONResponse{Code: "internal_error", Message: "failed to delete snapshot schedule"}, nil + } + } + + return oapi.DeleteInstanceSnapshotSchedule204Response{}, nil +} + func snapshotToOAPI(snapshot instances.Snapshot) oapi.Snapshot { kind := oapi.SnapshotKind(snapshot.Kind) sourceHypervisor := oapi.SnapshotSourceHypervisor(snapshot.SourceHypervisor) @@ -239,3 +358,30 @@ func snapshotToOAPI(snapshot instances.Snapshot) oapi.Snapshot { } return out } + +func snapshotScheduleToOAPI(schedule instances.SnapshotSchedule) oapi.SnapshotSchedule { + retention := oapi.SnapshotScheduleRetention{ + MaxCount: lo.ToPtr(schedule.Retention.MaxCount), + } + if schedule.Retention.MaxAge > 0 { + maxAge := schedule.Retention.MaxAge.String() + retention.MaxAge = &maxAge + } + + out := oapi.SnapshotSchedule{ + InstanceId: schedule.InstanceID, + Interval: schedule.Interval.String(), + Metadata: toOAPITags(schedule.Metadata), + Retention: retention, + NextRunAt: schedule.NextRunAt, + CreatedAt: schedule.CreatedAt, + UpdatedAt: schedule.UpdatedAt, + } + if schedule.NamePrefix != "" { + out.NamePrefix = lo.ToPtr(schedule.NamePrefix) + } + out.LastRunAt = schedule.LastRunAt + out.LastSnapshotId = schedule.LastSnapshotID + out.LastError = schedule.LastError + return out +} diff --git a/cmd/api/api/snapshots_test.go b/cmd/api/api/snapshots_test.go new file mode 100644 index 00000000..d0730133 --- /dev/null +++ b/cmd/api/api/snapshots_test.go @@ -0,0 +1,32 @@ +package api + +import ( + "testing" + "time" + + "github.com/kernel/hypeman/lib/instances" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSnapshotScheduleToOAPIPreservesZeroMaxCount(t *testing.T) { + t.Parallel() + + schedule := instances.SnapshotSchedule{ + InstanceID: "inst-1", + Interval: time.Hour, + Retention: instances.SnapshotScheduleRetention{ + MaxCount: 0, + MaxAge: 24 * time.Hour, + }, + NextRunAt: time.Now().UTC().Add(time.Hour), + CreatedAt: time.Now().UTC(), + UpdatedAt: time.Now().UTC(), + } + + out := snapshotScheduleToOAPI(schedule) + require.NotNil(t, out.Retention.MaxCount) + assert.Equal(t, 0, *out.Retention.MaxCount) + require.NotNil(t, out.Retention.MaxAge) + assert.Equal(t, "24h0m0s", *out.Retention.MaxAge) +} diff --git a/cmd/api/main.go b/cmd/api/main.go index bfdeaf8e..5bb18cc7 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -510,6 +510,29 @@ func run() error { } }) + // Snapshot schedule scheduler + if scheduleManager, ok := app.InstanceManager.(instances.SnapshotScheduleManager); ok { + const snapshotSchedulePollInterval = time.Minute + grp.Go(func() error { + ticker := time.NewTicker(snapshotSchedulePollInterval) + defer ticker.Stop() + + logger.Info("snapshot schedule scheduler started", "interval", snapshotSchedulePollInterval) + for { + select { + case <-gctx.Done(): + return nil + case <-ticker.C: + if err := scheduleManager.RunSnapshotSchedules(gctx); err != nil { + logger.Error("snapshot schedule run completed with errors", "error", err) + } + } + } + }) + } else { + logger.Warn("snapshot schedule manager unavailable; scheduled snapshots disabled") + } + err = grp.Wait() slog.Info("all goroutines finished") return err diff --git a/lib/instances/README.md b/lib/instances/README.md index 5a889d73..bf684bc9 100644 --- a/lib/instances/README.md +++ b/lib/instances/README.md @@ -108,6 +108,28 @@ Any State → Stopped - Don't prefault pages (lazy loading) - Parallel with TAP device setup +## Scheduled Snapshot Behavior + +- Schedules are configured per instance and persisted in the server data store (outside snapshot payloads). +- A background scheduler evaluates due schedules every minute. +- Each due run chooses snapshot behavior from current source state: + - `Running`/`Standby` sources use `Standby` snapshots. + - `Stopped` sources use `Stopped` snapshots. +- `Standby` runs from `Running` sources perform a brief pause/resume cycle during capture. +- The minimum interval is `1m`, but larger intervals are recommended for heavier or latency-sensitive workloads because running captures pause/resume the guest. +- Scheduled snapshot `name_prefix` is optional and capped at 47 chars so generated names stay within the 63-char snapshot name limit. +- New schedules establish cadence at `now + interval + deterministic jitter` derived from the instance ID. +- Updating only retention, metadata, or `name_prefix` preserves `next_run_at`; changing `interval` establishes a new cadence. +- Schedule runs advance to the next future interval (no backfill flood after downtime). +- Each schedule stores operational status: + - `next_run_at` + - `last_run_at` + - `last_snapshot_id` + - `last_error` +- Retention cleanup runs after successful scheduled snapshot creation and only affects scheduled snapshots for that instance. +- If an instance is deleted, its schedule is retained so retention can continue cleaning existing scheduled snapshots. +- Once the deleted instance has no scheduled snapshots left, the scheduler removes that schedule automatically. + ## Reference Handling Instances use OCI image references directly: diff --git a/lib/instances/errors.go b/lib/instances/errors.go index 33246fe5..c61f1039 100644 --- a/lib/instances/errors.go +++ b/lib/instances/errors.go @@ -29,4 +29,7 @@ var ( // ErrSnapshotNotFound is returned when a snapshot is not found. ErrSnapshotNotFound = errors.New("snapshot not found") + + // ErrSnapshotScheduleNotFound is returned when a snapshot schedule is not found. + ErrSnapshotScheduleNotFound = errors.New("snapshot schedule not found") ) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 44d9b162..c9b229f7 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -3,6 +3,7 @@ package instances import ( "context" "fmt" + "os" "sync" "time" @@ -88,6 +89,8 @@ type manager struct { meter metric.Meter tracer trace.Tracer now func() time.Time + writeFile func(string, []byte, os.FileMode) error + deleteSnapshotFn func(context.Context, string) error egressProxy *egressproxy.Service egressProxyServiceOptions egressproxy.ServiceOptions egressProxyMu sync.Mutex @@ -141,6 +144,7 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste vmStarters: vmStarters, defaultHypervisor: defaultHypervisor, now: time.Now, + writeFile: os.WriteFile, meter: meter, tracer: tracer, guestMemoryPolicy: policy, @@ -148,6 +152,7 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste compressionJobs: make(map[string]*compressionJob), nativeCodecPaths: make(map[string]string), } + m.deleteSnapshotFn = m.deleteSnapshot // Initialize metrics if meter is provided if meter != nil { diff --git a/lib/instances/snapshot_schedule.go b/lib/instances/snapshot_schedule.go new file mode 100644 index 00000000..541f6a6d --- /dev/null +++ b/lib/instances/snapshot_schedule.go @@ -0,0 +1,414 @@ +package instances + +import ( + "context" + "errors" + "fmt" + "os" + "sort" + "time" + + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/scheduledsnapshots" + "github.com/kernel/hypeman/lib/tags" +) + +type SnapshotScheduleRetention = scheduledsnapshots.Retention +type SnapshotSchedule = scheduledsnapshots.Schedule +type SetSnapshotScheduleRequest = scheduledsnapshots.SetRequest + +// SnapshotScheduleManager provides schedule operations in addition to core instance APIs. +type SnapshotScheduleManager interface { + SetSnapshotSchedule(ctx context.Context, instanceID string, req SetSnapshotScheduleRequest) (*SnapshotSchedule, error) + GetSnapshotSchedule(ctx context.Context, instanceID string) (*SnapshotSchedule, error) + DeleteSnapshotSchedule(ctx context.Context, instanceID string) error + RunSnapshotSchedules(ctx context.Context) error +} + +var _ SnapshotScheduleManager = (*manager)(nil) + +func (m *manager) SetSnapshotSchedule(ctx context.Context, instanceID string, req SetSnapshotScheduleRequest) (*SnapshotSchedule, error) { + if err := scheduledsnapshots.ValidateSetRequest(req, validateInstanceName); err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + + lock := m.getInstanceLock(instanceID) + lock.Lock() + defer lock.Unlock() + + if _, err := m.loadMetadata(instanceID); err != nil { + return nil, err + } + + now := m.now().UTC() + createdAt := now + nextRunAt := scheduledsnapshots.InitialNextRunAt(instanceID, req.Interval, now) + var lastRunAt *time.Time + var lastSnapshotID *string + var lastError *string + if existing, err := m.getSnapshotScheduleUnlocked(instanceID); err == nil { + createdAt = existing.CreatedAt + if existing.Interval == req.Interval && !existing.NextRunAt.IsZero() { + nextRunAt = existing.NextRunAt + } + if existing.LastRunAt != nil { + lastRunAtValue := *existing.LastRunAt + lastRunAt = &lastRunAtValue + } + if existing.LastSnapshotID != nil { + lastSnapshotIDValue := *existing.LastSnapshotID + lastSnapshotID = &lastSnapshotIDValue + } + if existing.LastError != nil { + lastErrorValue := *existing.LastError + lastError = &lastErrorValue + } + } else if !errors.Is(err, ErrSnapshotScheduleNotFound) { + return nil, err + } + + schedule := &SnapshotSchedule{ + InstanceID: instanceID, + Interval: req.Interval, + NamePrefix: req.NamePrefix, + Metadata: tags.Clone(req.Metadata), + Retention: req.Retention, + NextRunAt: nextRunAt, + LastRunAt: lastRunAt, + LastSnapshotID: lastSnapshotID, + LastError: lastError, + CreatedAt: createdAt, + UpdatedAt: now, + } + + if err := m.saveSnapshotScheduleUnlocked(schedule); err != nil { + return nil, err + } + + return schedule, nil +} + +func (m *manager) GetSnapshotSchedule(ctx context.Context, instanceID string) (*SnapshotSchedule, error) { + lock := m.getInstanceLock(instanceID) + lock.RLock() + defer lock.RUnlock() + + if _, err := m.loadMetadata(instanceID); err != nil { + return nil, err + } + return m.getSnapshotScheduleUnlocked(instanceID) +} + +func (m *manager) DeleteSnapshotSchedule(ctx context.Context, instanceID string) error { + lock := m.getInstanceLock(instanceID) + lock.Lock() + defer lock.Unlock() + + if _, err := m.loadMetadata(instanceID); err != nil { + return err + } + + err := os.Remove(m.paths.InstanceSnapshotSchedule(instanceID)) + if err == nil { + return nil + } + if os.IsNotExist(err) { + return ErrSnapshotScheduleNotFound + } + return fmt.Errorf("delete snapshot schedule: %w", err) +} + +func (m *manager) RunSnapshotSchedules(ctx context.Context) error { + log := logger.FromContext(ctx) + instanceIDs, err := m.listSnapshotScheduleInstanceIDs() + if err != nil { + return fmt.Errorf("list snapshot schedules: %w", err) + } + + var runErrs []error + + for _, instanceID := range instanceIDs { + if err := ctx.Err(); err != nil { + runErrs = append(runErrs, fmt.Errorf("run snapshot schedules: %w", err)) + break + } + + readNow := m.now().UTC() + lock := m.getInstanceLock(instanceID) + lock.RLock() + due, err := m.snapshotScheduleDueLocked(instanceID, readNow) + lock.RUnlock() + if err != nil { + if errors.Is(err, ErrSnapshotScheduleNotFound) { + continue + } + runErrs = append(runErrs, fmt.Errorf("instance %s: %w", instanceID, err)) + log.ErrorContext(ctx, "scheduled snapshot due-check failed", "instance_id", instanceID, "error", err) + continue + } + if !due { + continue + } + + lock.Lock() + runNow := m.now().UTC() + due, err = m.snapshotScheduleDueLocked(instanceID, runNow) + if err != nil { + lock.Unlock() + if errors.Is(err, ErrSnapshotScheduleNotFound) { + continue + } + runErrs = append(runErrs, fmt.Errorf("instance %s: %w", instanceID, err)) + log.ErrorContext(ctx, "scheduled snapshot due-check failed", "instance_id", instanceID, "error", err) + continue + } + if !due { + lock.Unlock() + continue + } + + err = m.runSnapshotScheduleForInstanceLocked(ctx, instanceID, runNow) + lock.Unlock() + if err != nil { + runErrs = append(runErrs, fmt.Errorf("instance %s: %w", instanceID, err)) + log.ErrorContext(ctx, "scheduled snapshot run failed", "instance_id", instanceID, "error", err) + } + } + + if len(runErrs) > 0 { + return errors.Join(runErrs...) + } + return nil +} + +func (m *manager) snapshotScheduleDueLocked(instanceID string, now time.Time) (bool, error) { + schedule, err := m.getSnapshotScheduleUnlocked(instanceID) + if err != nil { + return false, err + } + return !now.Before(schedule.NextRunAt), nil +} + +func (m *manager) runSnapshotScheduleForInstanceLocked(ctx context.Context, instanceID string, now time.Time) error { + schedule, err := m.getSnapshotScheduleUnlocked(instanceID) + if err != nil { + if errors.Is(err, ErrSnapshotScheduleNotFound) { + return nil + } + return err + } + if now.Before(schedule.NextRunAt) { + return nil + } + + runTime := now.UTC() + schedule.NextRunAt = scheduledsnapshots.NextRun(schedule.NextRunAt, schedule.Interval, runTime) + schedule.LastRunAt = &runTime + schedule.LastSnapshotID = nil + schedule.LastError = nil + schedule.UpdatedAt = runTime + + sourceMeta, sourceErr := m.loadMetadata(instanceID) + instanceMissing := errors.Is(sourceErr, ErrNotFound) + if sourceErr != nil && !instanceMissing { + return m.failScheduleRun(schedule, sourceErr, "load source metadata") + } + + if err := m.saveSnapshotScheduleUnlocked(schedule); err != nil { + return fmt.Errorf("save schedule before snapshot: %w", err) + } + + if !instanceMissing { + sourceState := m.toInstance(ctx, sourceMeta).State + snapshotKind, kindErr := scheduledSnapshotKindForState(sourceState) + if kindErr != nil { + return m.failScheduleRun(schedule, kindErr, "resolve scheduled snapshot kind") + } + + snapshot, runErr := m.createSnapshot(ctx, instanceID, CreateSnapshotRequest{ + Kind: snapshotKind, + Name: scheduledsnapshots.BuildSnapshotName(schedule.NamePrefix, runTime), + Tags: scheduledsnapshots.BuildSnapshotMetadata(instanceID, schedule.Metadata), + }) + if runErr != nil { + return m.failScheduleRun(schedule, runErr, "create scheduled snapshot") + } + + snapshotID := snapshot.Id + schedule.LastSnapshotID = &snapshotID + if err := m.saveSnapshotScheduleUnlocked(schedule); err != nil { + return fmt.Errorf("save schedule after snapshot: %w", err) + } + } + + cleanupErr := m.cleanupScheduledSnapshots(ctx, instanceID, schedule.Retention, runTime) + if cleanupErr != nil { + errMsg := cleanupErr.Error() + schedule.LastError = &errMsg + if saveErr := m.saveSnapshotScheduleUnlocked(schedule); saveErr != nil { + return fmt.Errorf("cleanup scheduled snapshots: %w; save schedule: %v", cleanupErr, saveErr) + } + return fmt.Errorf("cleanup scheduled snapshots: %w", cleanupErr) + } + + if instanceMissing { + scheduled, listErr := m.listScheduledSnapshotsByInstance(ctx, instanceID) + if listErr != nil { + errMsg := listErr.Error() + schedule.LastError = &errMsg + if saveErr := m.saveSnapshotScheduleUnlocked(schedule); saveErr != nil { + return fmt.Errorf("count scheduled snapshots: %w; save schedule: %v", listErr, saveErr) + } + return fmt.Errorf("count scheduled snapshots: %w", listErr) + } else { + remaining := len(scheduled) + shouldDeleteSchedule := remaining == 0 + if !shouldDeleteSchedule && schedule.Retention.MaxAge == 0 && schedule.Retention.MaxCount > 0 && remaining <= schedule.Retention.MaxCount { + // Count-only retention has converged for deleted instances: no future run can reduce count. + shouldDeleteSchedule = true + } + if shouldDeleteSchedule { + if err := os.Remove(m.paths.InstanceSnapshotSchedule(instanceID)); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("delete snapshot schedule after source deletion: %w", err) + } + return nil + } + } + } + + return nil +} + +func (m *manager) cleanupScheduledSnapshots(ctx context.Context, instanceID string, retention SnapshotScheduleRetention, now time.Time) error { + if retention.MaxCount == 0 && retention.MaxAge == 0 { + return nil + } + + scheduledSnapshots, err := m.listScheduledSnapshotsByInstance(ctx, instanceID) + if err != nil { + return err + } + + type candidate struct { + id string + createdAt time.Time + } + candidates := make([]candidate, 0, len(scheduledSnapshots)) + for _, snapshot := range scheduledSnapshots { + candidates = append(candidates, candidate{id: snapshot.Id, createdAt: snapshot.CreatedAt}) + } + + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].createdAt.After(candidates[j].createdAt) + }) + + deleteIDs := make(map[string]struct{}) + if retention.MaxCount > 0 && len(candidates) > retention.MaxCount { + for _, candidate := range candidates[retention.MaxCount:] { + deleteIDs[candidate.id] = struct{}{} + } + } + if retention.MaxAge > 0 { + cutoff := now.Add(-retention.MaxAge) + for _, candidate := range candidates { + if candidate.createdAt.Before(cutoff) { + deleteIDs[candidate.id] = struct{}{} + } + } + } + + if len(deleteIDs) == 0 { + return nil + } + + var errs []error + for snapshotID := range deleteIDs { + if err := m.deleteSnapshotFn(ctx, snapshotID); err != nil { + if errors.Is(err, ErrSnapshotNotFound) { + continue + } + errs = append(errs, fmt.Errorf("delete snapshot %s: %w", snapshotID, err)) + } + } + if len(errs) > 0 { + return errors.Join(errs...) + } + + return nil +} + +func (m *manager) failScheduleRun(schedule *SnapshotSchedule, err error, context string) error { + errMsg := err.Error() + schedule.LastError = &errMsg + if saveErr := m.saveSnapshotScheduleUnlocked(schedule); saveErr != nil { + return fmt.Errorf("%s: %w; save schedule: %v", context, err, saveErr) + } + return fmt.Errorf("%s: %w", context, err) +} + +func (m *manager) getSnapshotScheduleUnlocked(instanceID string) (*SnapshotSchedule, error) { + content, err := os.ReadFile(m.paths.InstanceSnapshotSchedule(instanceID)) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrSnapshotScheduleNotFound + } + return nil, fmt.Errorf("read snapshot schedule: %w", err) + } + + schedule, err := scheduledsnapshots.UnmarshalSchedule(content) + if err != nil { + return nil, err + } + return schedule, nil +} + +func (m *manager) saveSnapshotScheduleUnlocked(schedule *SnapshotSchedule) error { + content, err := scheduledsnapshots.MarshalSchedule(schedule) + if err != nil { + return fmt.Errorf("marshal snapshot schedule: %w", err) + } + + if err := os.MkdirAll(m.paths.SnapshotSchedulesDir(), 0755); err != nil { + return fmt.Errorf("create snapshot schedules directory: %w", err) + } + writeFile := m.writeFile + if writeFile == nil { + writeFile = os.WriteFile + } + if err := writeFile(m.paths.InstanceSnapshotSchedule(schedule.InstanceID), content, 0644); err != nil { + return fmt.Errorf("write snapshot schedule: %w", err) + } + + return nil +} + +func (m *manager) listSnapshotScheduleInstanceIDs() ([]string, error) { + return scheduledsnapshots.ListInstanceIDs(m.paths.SnapshotSchedulesDir()) +} + +func (m *manager) listScheduledSnapshotsByInstance(ctx context.Context, instanceID string) ([]Snapshot, error) { + filter := &ListSnapshotsFilter{SourceInstanceID: &instanceID} + snapshots, err := m.listSnapshots(ctx, filter) + if err != nil { + return nil, fmt.Errorf("list snapshots: %w", err) + } + + scheduledSnapshots := make([]Snapshot, 0, len(snapshots)) + for _, snapshot := range snapshots { + if scheduledsnapshots.IsScheduledSnapshot(snapshot.Tags, instanceID) { + scheduledSnapshots = append(scheduledSnapshots, snapshot) + } + } + return scheduledSnapshots, nil +} + +func scheduledSnapshotKindForState(state State) (SnapshotKind, error) { + switch state { + case StateStopped: + return SnapshotKindStopped, nil + case StateRunning, StateStandby: + return SnapshotKindStandby, nil + default: + return "", fmt.Errorf("%w: scheduled snapshot requires source in %s, %s, or %s, got %s", ErrInvalidState, StateRunning, StateStandby, StateStopped, state) + } +} diff --git a/lib/instances/snapshot_schedule_test.go b/lib/instances/snapshot_schedule_test.go new file mode 100644 index 00000000..06d50e5b --- /dev/null +++ b/lib/instances/snapshot_schedule_test.go @@ -0,0 +1,658 @@ +package instances + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/kernel/hypeman/lib/scheduledsnapshots" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSnapshotScheduleSetGetDelete(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-schedule-src" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, "snapshot-schedule-src", hvType) + + schedule, err := mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: 2 * time.Hour, + NamePrefix: "nightly", + Metadata: map[string]string{"env": "test"}, + Retention: SnapshotScheduleRetention{ + MaxCount: 5, + }, + }) + require.NoError(t, err) + require.Equal(t, sourceID, schedule.InstanceID) + require.Equal(t, 2*time.Hour, schedule.Interval) + require.Equal(t, "nightly", schedule.NamePrefix) + require.Equal(t, 5, schedule.Retention.MaxCount) + assert.GreaterOrEqual(t, schedule.NextRunAt, mgr.now().UTC().Add(2*time.Hour)) + assert.Less(t, schedule.NextRunAt, mgr.now().UTC().Add(2*time.Hour).Add(5*time.Minute)) + + loaded, err := mgr.GetSnapshotSchedule(ctx, sourceID) + require.NoError(t, err) + assert.Equal(t, schedule.InstanceID, loaded.InstanceID) + assert.Equal(t, schedule.Interval, loaded.Interval) + assert.Equal(t, schedule.Retention.MaxCount, loaded.Retention.MaxCount) + assert.Equal(t, "test", loaded.Metadata["env"]) + + require.NoError(t, mgr.DeleteSnapshotSchedule(ctx, sourceID)) + + _, err = mgr.GetSnapshotSchedule(ctx, sourceID) + require.Error(t, err) + assert.ErrorIs(t, err, ErrSnapshotScheduleNotFound) +} + +func TestSnapshotScheduleUpdatePreservesOperationalHistory(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-schedule-update-history-src" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, hvType) + + created, err := mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + Retention: SnapshotScheduleRetention{ + MaxCount: 2, + }, + }) + require.NoError(t, err) + + expectedLastRunAt := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Second) + expectedLastSnapshotID := "snapshot-123" + expectedLastError := "last run failed" + + lock := mgr.getInstanceLock(sourceID) + lock.Lock() + schedule, err := mgr.getSnapshotScheduleUnlocked(sourceID) + require.NoError(t, err) + schedule.LastRunAt = &expectedLastRunAt + schedule.LastSnapshotID = &expectedLastSnapshotID + schedule.LastError = &expectedLastError + require.NoError(t, mgr.saveSnapshotScheduleUnlocked(schedule)) + lock.Unlock() + + updated, err := mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: 2 * time.Hour, + NamePrefix: "nightly", + Retention: SnapshotScheduleRetention{ + MaxCount: 5, + }, + }) + require.NoError(t, err) + + require.NotNil(t, updated.LastRunAt) + assert.Equal(t, expectedLastRunAt, *updated.LastRunAt) + require.NotNil(t, updated.LastSnapshotID) + assert.Equal(t, expectedLastSnapshotID, *updated.LastSnapshotID) + require.NotNil(t, updated.LastError) + assert.Equal(t, expectedLastError, *updated.LastError) + assert.Equal(t, created.CreatedAt, updated.CreatedAt) +} + +func TestSnapshotScheduleConfigOnlyUpdatePreservesNextRunAt(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + now := time.Date(2026, 3, 24, 10, 0, 0, 0, time.UTC) + mgr.now = func() time.Time { return now } + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-schedule-preserve-next-run-src" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, hvType) + + created, err := mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + Retention: SnapshotScheduleRetention{ + MaxCount: 2, + }, + }) + require.NoError(t, err) + + updatedNow := now.Add(30 * time.Minute) + mgr.now = func() time.Time { return updatedNow } + + updated, err := mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + NamePrefix: "nightly", + Metadata: map[string]string{"env": "prod"}, + Retention: SnapshotScheduleRetention{ + MaxCount: 5, + }, + }) + require.NoError(t, err) + assert.Equal(t, created.NextRunAt, updated.NextRunAt) + assert.Equal(t, updatedNow, updated.UpdatedAt) +} + +func TestSnapshotScheduleIntervalUpdateRecomputesNextRunAt(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + now := time.Date(2026, 3, 24, 10, 0, 0, 0, time.UTC) + mgr.now = func() time.Time { return now } + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-schedule-recompute-next-run-src" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, hvType) + + created, err := mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + Retention: SnapshotScheduleRetention{ + MaxCount: 2, + }, + }) + require.NoError(t, err) + + updatedNow := now.Add(30 * time.Minute) + mgr.now = func() time.Time { return updatedNow } + + updated, err := mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: 2 * time.Hour, + Retention: SnapshotScheduleRetention{ + MaxCount: 2, + }, + }) + require.NoError(t, err) + assert.NotEqual(t, created.NextRunAt, updated.NextRunAt) + assert.Equal(t, scheduledsnapshots.InitialNextRunAt(sourceID, 2*time.Hour, updatedNow), updated.NextRunAt) +} + +func TestSnapshotScheduleUsesStoppedSnapshotWhenSourceIsStopped(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-schedule-auto-kind-stopped-src" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, hvType) + + _, err := mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + Retention: SnapshotScheduleRetention{ + MaxCount: 1, + }, + }) + require.NoError(t, err) + + markSnapshotScheduleDue(t, mgr, sourceID) + require.NoError(t, mgr.RunSnapshotSchedules(ctx)) + + snaps, err := mgr.ListSnapshots(ctx, &ListSnapshotsFilter{SourceInstanceID: &sourceID}) + require.NoError(t, err) + + foundScheduled := false + for _, snapshot := range snaps { + if !scheduledsnapshots.IsScheduledSnapshot(snapshot.Tags, sourceID) { + continue + } + foundScheduled = true + assert.Equal(t, SnapshotKindStopped, snapshot.Kind) + } + require.True(t, foundScheduled, "expected at least one scheduled snapshot") +} + +func TestRunSnapshotSchedulesCreatesSnapshotAndAppliesRetention(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-schedule-run-src" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, "snapshot-schedule-run-src", hvType) + + older, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "older-scheduled", + Tags: map[string]string{ + scheduledsnapshots.MetadataKeyScheduled: "true", + scheduledsnapshots.MetadataKeySourceInstanceID: sourceID, + }, + }) + require.NoError(t, err) + + time.Sleep(1100 * time.Millisecond) + + newer, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "newer-scheduled", + Tags: map[string]string{ + scheduledsnapshots.MetadataKeyScheduled: "true", + scheduledsnapshots.MetadataKeySourceInstanceID: sourceID, + }, + }) + require.NoError(t, err) + + _, err = mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "manual-snapshot", + }) + require.NoError(t, err) + + _, err = mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + NamePrefix: "nightly", + Retention: SnapshotScheduleRetention{ + MaxCount: 2, + }, + }) + require.NoError(t, err) + + markSnapshotScheduleDue(t, mgr, sourceID) + + require.NoError(t, mgr.RunSnapshotSchedules(ctx)) + + snaps, err := mgr.ListSnapshots(ctx, &ListSnapshotsFilter{SourceInstanceID: &sourceID}) + require.NoError(t, err) + + scheduledIDs := make(map[string]struct{}) + manualCount := 0 + for _, snapshot := range snaps { + if scheduledsnapshots.IsScheduledSnapshot(snapshot.Tags, sourceID) { + scheduledIDs[snapshot.Id] = struct{}{} + } else { + manualCount++ + } + } + + assert.Len(t, scheduledIDs, 2, "retention should keep only two scheduled snapshots") + assert.NotContains(t, scheduledIDs, older.Id, "oldest scheduled snapshot should be cleaned up") + assert.Contains(t, scheduledIDs, newer.Id, "newer pre-existing scheduled snapshot should be retained") + assert.Equal(t, 1, manualCount, "manual snapshots must not be auto-cleaned") + + schedule, err := mgr.GetSnapshotSchedule(ctx, sourceID) + require.NoError(t, err) + require.NotNil(t, schedule.LastRunAt) + require.NotNil(t, schedule.LastSnapshotID) + assert.Nil(t, schedule.LastError) +} + +func TestSnapshotScheduleRequiresRetentionPolicy(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-schedule-validate-src" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, "snapshot-schedule-validate-src", hvType) + + _, err := mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + }) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) +} + +func TestSnapshotScheduleNamePrefixLengthValidation(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-schedule-prefix-src" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, hvType) + + tooLong := strings.Repeat("a", scheduledsnapshots.MaxNamePrefixLength+1) + _, err := mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + NamePrefix: tooLong, + Retention: SnapshotScheduleRetention{ + MaxCount: 1, + }, + }) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "name_prefix must be at most") + + atLimit := strings.Repeat("a", scheduledsnapshots.MaxNamePrefixLength) + _, err = mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + NamePrefix: atLimit, + Retention: SnapshotScheduleRetention{ + MaxCount: 1, + }, + }) + require.NoError(t, err) +} + +func TestDeleteInstanceKeepsScheduleUntilScheduledSnapshotsAreGone(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-schedule-delete-src" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, "snapshot-schedule-delete-src", hvType) + + _, err := mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + Retention: SnapshotScheduleRetention{ + MaxAge: 24 * time.Hour, + }, + }) + require.NoError(t, err) + + scheduledSnapshot, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "scheduled-before-delete", + Tags: map[string]string{ + scheduledsnapshots.MetadataKeyScheduled: "true", + scheduledsnapshots.MetadataKeySourceInstanceID: sourceID, + }, + }) + require.NoError(t, err) + require.FileExists(t, mgr.paths.InstanceSnapshotSchedule(sourceID)) + + require.NoError(t, mgr.DeleteInstance(ctx, sourceID)) + require.FileExists(t, mgr.paths.InstanceSnapshotSchedule(sourceID)) + + markSnapshotScheduleDue(t, mgr, sourceID) + require.NoError(t, mgr.RunSnapshotSchedules(ctx)) + require.FileExists(t, mgr.paths.InstanceSnapshotSchedule(sourceID)) + + require.NoError(t, mgr.DeleteSnapshot(ctx, scheduledSnapshot.Id)) + + markSnapshotScheduleDue(t, mgr, sourceID) + require.NoError(t, mgr.RunSnapshotSchedules(ctx)) + + _, statErr := os.Stat(mgr.paths.InstanceSnapshotSchedule(sourceID)) + require.Error(t, statErr) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestDeleteInstanceWithCountOnlyRetentionRemovesConvergedSchedule(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-schedule-delete-count-only-src" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, hvType) + + _, err := mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + Retention: SnapshotScheduleRetention{ + MaxCount: 3, + }, + }) + require.NoError(t, err) + + _, err = mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "scheduled-before-delete-count-only", + Tags: map[string]string{ + scheduledsnapshots.MetadataKeyScheduled: "true", + scheduledsnapshots.MetadataKeySourceInstanceID: sourceID, + }, + }) + require.NoError(t, err) + + require.NoError(t, mgr.DeleteInstance(ctx, sourceID)) + require.FileExists(t, mgr.paths.InstanceSnapshotSchedule(sourceID)) + + markSnapshotScheduleDue(t, mgr, sourceID) + require.NoError(t, mgr.RunSnapshotSchedules(ctx)) + + _, statErr := os.Stat(mgr.paths.InstanceSnapshotSchedule(sourceID)) + require.Error(t, statErr) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestRunSnapshotSchedulesAggregatesErrorsAcrossInstances(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceA := "snapshot-schedule-error-src-a" + sourceB := "snapshot-schedule-error-src-b" + createStoppedSnapshotSourceFixture(t, mgr, sourceA, sourceA, hvType) + createStoppedSnapshotSourceFixture(t, mgr, sourceB, sourceB, hvType) + + _, err := mgr.SetSnapshotSchedule(ctx, sourceA, SetSnapshotScheduleRequest{ + Interval: time.Hour, + Retention: SnapshotScheduleRetention{ + MaxCount: 1, + }, + }) + require.NoError(t, err) + _, err = mgr.SetSnapshotSchedule(ctx, sourceB, SetSnapshotScheduleRequest{ + Interval: time.Hour, + Retention: SnapshotScheduleRetention{ + MaxCount: 1, + }, + }) + require.NoError(t, err) + + markSnapshotScheduleDue(t, mgr, sourceA) + markSnapshotScheduleDue(t, mgr, sourceB) + + require.NoError(t, os.WriteFile(mgr.paths.InstanceSnapshotSchedule(sourceA), []byte("{"), 0644)) + require.NoError(t, os.WriteFile(mgr.paths.InstanceSnapshotSchedule(sourceB), []byte("{"), 0644)) + + err = mgr.RunSnapshotSchedules(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), sourceA) + assert.Contains(t, err.Error(), sourceB) +} + +func TestRunSnapshotSchedulesSaveFailureAfterSnapshotDoesNotDuplicate(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + now := time.Date(2026, 3, 24, 10, 0, 0, 0, time.UTC) + mgr.now = func() time.Time { return now } + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-schedule-save-failure-src" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, hvType) + + _, err := mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + Retention: SnapshotScheduleRetention{ + MaxCount: 2, + }, + }) + require.NoError(t, err) + + markSnapshotScheduleDue(t, mgr, sourceID) + + schedulePath := mgr.paths.InstanceSnapshotSchedule(sourceID) + originalWriteFile := mgr.writeFile + writeCount := 0 + mgr.writeFile = func(name string, data []byte, perm os.FileMode) error { + if name == schedulePath { + writeCount++ + if writeCount == 2 { + return errors.New("injected schedule save failure") + } + } + return originalWriteFile(name, data, perm) + } + defer func() { + mgr.writeFile = originalWriteFile + }() + + err = mgr.RunSnapshotSchedules(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "save schedule after snapshot") + + snapshots, err := mgr.ListSnapshots(ctx, &ListSnapshotsFilter{SourceInstanceID: &sourceID}) + require.NoError(t, err) + + scheduledCount := 0 + for _, snapshot := range snapshots { + if scheduledsnapshots.IsScheduledSnapshot(snapshot.Tags, sourceID) { + scheduledCount++ + } + } + assert.Equal(t, 1, scheduledCount) + + schedule, err := mgr.GetSnapshotSchedule(ctx, sourceID) + require.NoError(t, err) + require.NotNil(t, schedule.LastRunAt) + assert.Equal(t, now.Add(59*time.Minute), schedule.NextRunAt) + assert.Nil(t, schedule.LastSnapshotID) + assert.Nil(t, schedule.LastError) + + mgr.writeFile = originalWriteFile + + err = mgr.RunSnapshotSchedules(ctx) + require.NoError(t, err) + + snapshots, err = mgr.ListSnapshots(ctx, &ListSnapshotsFilter{SourceInstanceID: &sourceID}) + require.NoError(t, err) + scheduledCount = 0 + for _, snapshot := range snapshots { + if scheduledsnapshots.IsScheduledSnapshot(snapshot.Tags, sourceID) { + scheduledCount++ + } + } + assert.Equal(t, 1, scheduledCount) +} + +func TestRunSnapshotSchedulesCleanupFailurePreservesLastSnapshotIDAndDoesNotDuplicate(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + now := time.Date(2026, 3, 24, 10, 0, 0, 0, time.UTC) + mgr.now = func() time.Time { return now } + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-schedule-cleanup-failure-src" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, hvType) + + older, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "older-scheduled", + Tags: map[string]string{ + scheduledsnapshots.MetadataKeyScheduled: "true", + scheduledsnapshots.MetadataKeySourceInstanceID: sourceID, + }, + }) + require.NoError(t, err) + + originalDeleteSnapshotFn := mgr.deleteSnapshotFn + injectedDeleteErr := errors.New("injected snapshot delete failure") + mgr.deleteSnapshotFn = func(ctx context.Context, snapshotID string) error { + if snapshotID == older.Id { + return injectedDeleteErr + } + return originalDeleteSnapshotFn(ctx, snapshotID) + } + defer func() { + mgr.deleteSnapshotFn = originalDeleteSnapshotFn + }() + + _, err = mgr.SetSnapshotSchedule(ctx, sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + Retention: SnapshotScheduleRetention{ + MaxCount: 1, + }, + }) + require.NoError(t, err) + + markSnapshotScheduleDue(t, mgr, sourceID) + + err = mgr.RunSnapshotSchedules(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "cleanup scheduled snapshots") + assert.ErrorIs(t, err, injectedDeleteErr) + + schedule, err := mgr.GetSnapshotSchedule(ctx, sourceID) + require.NoError(t, err) + require.NotNil(t, schedule.LastSnapshotID) + require.NotNil(t, schedule.LastError) + + snapshots, err := mgr.ListSnapshots(ctx, &ListSnapshotsFilter{SourceInstanceID: &sourceID}) + require.NoError(t, err) + scheduledCount := 0 + lastSnapshotFound := false + for _, snapshot := range snapshots { + if !scheduledsnapshots.IsScheduledSnapshot(snapshot.Tags, sourceID) { + continue + } + scheduledCount++ + if snapshot.Id == *schedule.LastSnapshotID { + lastSnapshotFound = true + } + } + assert.Equal(t, 2, scheduledCount) + assert.True(t, lastSnapshotFound) + require.DirExists(t, mgr.paths.SnapshotDir(older.Id)) + + err = mgr.RunSnapshotSchedules(ctx) + require.NoError(t, err) + + snapshots, err = mgr.ListSnapshots(ctx, &ListSnapshotsFilter{SourceInstanceID: &sourceID}) + require.NoError(t, err) + scheduledCount = 0 + for _, snapshot := range snapshots { + if scheduledsnapshots.IsScheduledSnapshot(snapshot.Tags, sourceID) { + scheduledCount++ + } + } + assert.Equal(t, 2, scheduledCount) + require.DirExists(t, mgr.paths.SnapshotDir(older.Id)) +} + +func TestRunSnapshotSchedulesStopsOnCanceledContext(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-schedule-canceled-context-src" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, hvType) + + _, err := mgr.SetSnapshotSchedule(context.Background(), sourceID, SetSnapshotScheduleRequest{ + Interval: time.Hour, + Retention: SnapshotScheduleRetention{ + MaxCount: 1, + }, + }) + require.NoError(t, err) + + markSnapshotScheduleDue(t, mgr, sourceID) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err = mgr.RunSnapshotSchedules(ctx) + require.Error(t, err) + assert.ErrorIs(t, err, context.Canceled) + + snapshots, err := mgr.ListSnapshots(context.Background(), &ListSnapshotsFilter{SourceInstanceID: &sourceID}) + require.NoError(t, err) + + for _, snapshot := range snapshots { + assert.False(t, scheduledsnapshots.IsScheduledSnapshot(snapshot.Tags, sourceID), "canceled context should prevent scheduled snapshot runs") + } +} + +func markSnapshotScheduleDue(t *testing.T, mgr *manager, instanceID string) { + t.Helper() + lock := mgr.getInstanceLock(instanceID) + lock.Lock() + defer lock.Unlock() + + schedule, err := mgr.getSnapshotScheduleUnlocked(instanceID) + require.NoError(t, err) + schedule.NextRunAt = mgr.now().UTC().Add(-time.Minute) + schedule.UpdatedAt = mgr.now().UTC() + require.NoError(t, mgr.saveSnapshotScheduleUnlocked(schedule)) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 7a07110f..966d4922 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1082,6 +1082,21 @@ type RestoreSnapshotRequest struct { // Standby snapshots must restore with their original hypervisor. type RestoreSnapshotRequestTargetHypervisor string +// SetSnapshotScheduleRequest defines model for SetSnapshotScheduleRequest. +type SetSnapshotScheduleRequest struct { + // Interval Snapshot interval (Go duration format, minimum 1m). + Interval string `json:"interval"` + + // Metadata User-defined key-value tags. + Metadata *Tags `json:"metadata,omitempty"` + + // NamePrefix Optional prefix for auto-generated scheduled snapshot names (max 47 chars). + NamePrefix *string `json:"name_prefix"` + + // Retention Automatic cleanup policy for scheduled snapshots. + Retention SnapshotScheduleRetention `json:"retention"` +} + // Snapshot defines model for Snapshot. type Snapshot struct { // CompressedSizeBytes Compressed memory payload size in bytes @@ -1154,6 +1169,51 @@ type SnapshotPolicy struct { Compression *SnapshotCompressionConfig `json:"compression,omitempty"` } +// SnapshotSchedule defines model for SnapshotSchedule. +type SnapshotSchedule struct { + // CreatedAt Schedule creation timestamp. + CreatedAt time.Time `json:"created_at"` + + // InstanceId Source instance ID. + InstanceId string `json:"instance_id"` + + // Interval Snapshot interval (Go duration format). + Interval string `json:"interval"` + + // LastError Last schedule run error, if any. + LastError *string `json:"last_error"` + + // LastRunAt Last schedule execution time. + LastRunAt *time.Time `json:"last_run_at"` + + // LastSnapshotId Snapshot ID produced by the last successful run. + LastSnapshotId *string `json:"last_snapshot_id"` + + // Metadata User-defined key-value tags. + Metadata *Tags `json:"metadata,omitempty"` + + // NamePrefix Optional prefix used for generated scheduled snapshot names. + NamePrefix *string `json:"name_prefix"` + + // NextRunAt Next scheduled run time. + NextRunAt time.Time `json:"next_run_at"` + + // Retention Automatic cleanup policy for scheduled snapshots. + Retention SnapshotScheduleRetention `json:"retention"` + + // UpdatedAt Schedule update timestamp. + UpdatedAt time.Time `json:"updated_at"` +} + +// SnapshotScheduleRetention Automatic cleanup policy for scheduled snapshots. +type SnapshotScheduleRetention struct { + // MaxAge Delete scheduled snapshots older than this duration (Go duration format). + MaxAge *string `json:"max_age,omitempty"` + + // MaxCount Keep at most this many scheduled snapshots for the instance (0 disables count-based cleanup). + MaxCount *int `json:"max_count,omitempty"` +} + // SnapshotTargetState Target state when restoring or forking from a snapshot type SnapshotTargetState string @@ -1404,6 +1464,9 @@ type UpdateInstanceJSONRequestBody = UpdateInstanceRequest // ForkInstanceJSONRequestBody defines body for ForkInstance for application/json ContentType. type ForkInstanceJSONRequestBody = ForkInstanceRequest +// SetInstanceSnapshotScheduleJSONRequestBody defines body for SetInstanceSnapshotSchedule for application/json ContentType. +type SetInstanceSnapshotScheduleJSONRequestBody = SetSnapshotScheduleRequest + // CreateInstanceSnapshotJSONRequestBody defines body for CreateInstanceSnapshot for application/json ContentType. type CreateInstanceSnapshotJSONRequestBody = CreateSnapshotRequest @@ -1594,6 +1657,17 @@ type ClientInterface interface { // RestoreInstance request RestoreInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // DeleteInstanceSnapshotSchedule request + DeleteInstanceSnapshotSchedule(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + + // GetInstanceSnapshotSchedule request + GetInstanceSnapshotSchedule(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + + // SetInstanceSnapshotScheduleWithBody request with any body + SetInstanceSnapshotScheduleWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + SetInstanceSnapshotSchedule(ctx context.Context, id string, body SetInstanceSnapshotScheduleJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // CreateInstanceSnapshotWithBody request with any body CreateInstanceSnapshotWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -2067,6 +2141,54 @@ func (c *Client) RestoreInstance(ctx context.Context, id string, reqEditors ...R return c.Client.Do(req) } +func (c *Client) DeleteInstanceSnapshotSchedule(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewDeleteInstanceSnapshotScheduleRequest(c.Server, id) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) GetInstanceSnapshotSchedule(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetInstanceSnapshotScheduleRequest(c.Server, id) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) SetInstanceSnapshotScheduleWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewSetInstanceSnapshotScheduleRequestWithBody(c.Server, id, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) SetInstanceSnapshotSchedule(ctx context.Context, id string, body SetInstanceSnapshotScheduleJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewSetInstanceSnapshotScheduleRequest(c.Server, id, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) CreateInstanceSnapshotWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewCreateInstanceSnapshotRequestWithBody(c.Server, id, contentType, body) if err != nil { @@ -3519,6 +3641,121 @@ func NewRestoreInstanceRequest(server string, id string) (*http.Request, error) return req, nil } +// NewDeleteInstanceSnapshotScheduleRequest generates requests for DeleteInstanceSnapshotSchedule +func NewDeleteInstanceSnapshotScheduleRequest(server string, id string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/instances/%s/snapshot-schedule", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("DELETE", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewGetInstanceSnapshotScheduleRequest generates requests for GetInstanceSnapshotSchedule +func NewGetInstanceSnapshotScheduleRequest(server string, id string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/instances/%s/snapshot-schedule", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewSetInstanceSnapshotScheduleRequest calls the generic SetInstanceSnapshotSchedule builder with application/json body +func NewSetInstanceSnapshotScheduleRequest(server string, id string, body SetInstanceSnapshotScheduleJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewSetInstanceSnapshotScheduleRequestWithBody(server, id, "application/json", bodyReader) +} + +// NewSetInstanceSnapshotScheduleRequestWithBody generates requests for SetInstanceSnapshotSchedule with any type of body +func NewSetInstanceSnapshotScheduleRequestWithBody(server string, id string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/instances/%s/snapshot-schedule", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("PUT", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewCreateInstanceSnapshotRequest calls the generic CreateInstanceSnapshot builder with application/json body func NewCreateInstanceSnapshotRequest(server string, id string, body CreateInstanceSnapshotJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -4608,6 +4845,17 @@ type ClientWithResponsesInterface interface { // RestoreInstanceWithResponse request RestoreInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*RestoreInstanceResponse, error) + // DeleteInstanceSnapshotScheduleWithResponse request + DeleteInstanceSnapshotScheduleWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*DeleteInstanceSnapshotScheduleResponse, error) + + // GetInstanceSnapshotScheduleWithResponse request + GetInstanceSnapshotScheduleWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetInstanceSnapshotScheduleResponse, error) + + // SetInstanceSnapshotScheduleWithBodyWithResponse request with any body + SetInstanceSnapshotScheduleWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*SetInstanceSnapshotScheduleResponse, error) + + SetInstanceSnapshotScheduleWithResponse(ctx context.Context, id string, body SetInstanceSnapshotScheduleJSONRequestBody, reqEditors ...RequestEditorFn) (*SetInstanceSnapshotScheduleResponse, error) + // CreateInstanceSnapshotWithBodyWithResponse request with any body CreateInstanceSnapshotWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateInstanceSnapshotResponse, error) @@ -5344,6 +5592,78 @@ func (r RestoreInstanceResponse) StatusCode() int { return 0 } +type DeleteInstanceSnapshotScheduleResponse struct { + Body []byte + HTTPResponse *http.Response + JSON404 *Error + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r DeleteInstanceSnapshotScheduleResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r DeleteInstanceSnapshotScheduleResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type GetInstanceSnapshotScheduleResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *SnapshotSchedule + JSON404 *Error + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r GetInstanceSnapshotScheduleResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetInstanceSnapshotScheduleResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type SetInstanceSnapshotScheduleResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *SnapshotSchedule + JSON400 *Error + JSON404 *Error + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r SetInstanceSnapshotScheduleResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r SetInstanceSnapshotScheduleResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type CreateInstanceSnapshotResponse struct { Body []byte HTTPResponse *http.Response @@ -6132,6 +6452,41 @@ func (c *ClientWithResponses) RestoreInstanceWithResponse(ctx context.Context, i return ParseRestoreInstanceResponse(rsp) } +// DeleteInstanceSnapshotScheduleWithResponse request returning *DeleteInstanceSnapshotScheduleResponse +func (c *ClientWithResponses) DeleteInstanceSnapshotScheduleWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*DeleteInstanceSnapshotScheduleResponse, error) { + rsp, err := c.DeleteInstanceSnapshotSchedule(ctx, id, reqEditors...) + if err != nil { + return nil, err + } + return ParseDeleteInstanceSnapshotScheduleResponse(rsp) +} + +// GetInstanceSnapshotScheduleWithResponse request returning *GetInstanceSnapshotScheduleResponse +func (c *ClientWithResponses) GetInstanceSnapshotScheduleWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetInstanceSnapshotScheduleResponse, error) { + rsp, err := c.GetInstanceSnapshotSchedule(ctx, id, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetInstanceSnapshotScheduleResponse(rsp) +} + +// SetInstanceSnapshotScheduleWithBodyWithResponse request with arbitrary body returning *SetInstanceSnapshotScheduleResponse +func (c *ClientWithResponses) SetInstanceSnapshotScheduleWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*SetInstanceSnapshotScheduleResponse, error) { + rsp, err := c.SetInstanceSnapshotScheduleWithBody(ctx, id, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseSetInstanceSnapshotScheduleResponse(rsp) +} + +func (c *ClientWithResponses) SetInstanceSnapshotScheduleWithResponse(ctx context.Context, id string, body SetInstanceSnapshotScheduleJSONRequestBody, reqEditors ...RequestEditorFn) (*SetInstanceSnapshotScheduleResponse, error) { + rsp, err := c.SetInstanceSnapshotSchedule(ctx, id, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseSetInstanceSnapshotScheduleResponse(rsp) +} + // CreateInstanceSnapshotWithBodyWithResponse request with arbitrary body returning *CreateInstanceSnapshotResponse func (c *ClientWithResponses) CreateInstanceSnapshotWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateInstanceSnapshotResponse, error) { rsp, err := c.CreateInstanceSnapshotWithBody(ctx, id, contentType, body, reqEditors...) @@ -7533,8 +7888,128 @@ func ParseRestoreInstanceResponse(rsp *http.Response) (*RestoreInstanceResponse, return response, nil } -// ParseCreateInstanceSnapshotResponse parses an HTTP response from a CreateInstanceSnapshotWithResponse call -func ParseCreateInstanceSnapshotResponse(rsp *http.Response) (*CreateInstanceSnapshotResponse, error) { +// ParseDeleteInstanceSnapshotScheduleResponse parses an HTTP response from a DeleteInstanceSnapshotScheduleWithResponse call +func ParseDeleteInstanceSnapshotScheduleResponse(rsp *http.Response) (*DeleteInstanceSnapshotScheduleResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &DeleteInstanceSnapshotScheduleResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + +// ParseGetInstanceSnapshotScheduleResponse parses an HTTP response from a GetInstanceSnapshotScheduleWithResponse call +func ParseGetInstanceSnapshotScheduleResponse(rsp *http.Response) (*GetInstanceSnapshotScheduleResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetInstanceSnapshotScheduleResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest SnapshotSchedule + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + +// ParseSetInstanceSnapshotScheduleResponse parses an HTTP response from a SetInstanceSnapshotScheduleWithResponse call +func ParseSetInstanceSnapshotScheduleResponse(rsp *http.Response) (*SetInstanceSnapshotScheduleResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &SetInstanceSnapshotScheduleResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest SnapshotSchedule + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + +// ParseCreateInstanceSnapshotResponse parses an HTTP response from a CreateInstanceSnapshotWithResponse call +func ParseCreateInstanceSnapshotResponse(rsp *http.Response) (*CreateInstanceSnapshotResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) defer func() { _ = rsp.Body.Close() }() if err != nil { @@ -8535,6 +9010,15 @@ type ServerInterface interface { // Restore instance from standby // (POST /instances/{id}/restore) RestoreInstance(w http.ResponseWriter, r *http.Request, id string) + // Delete snapshot schedule for an instance + // (DELETE /instances/{id}/snapshot-schedule) + DeleteInstanceSnapshotSchedule(w http.ResponseWriter, r *http.Request, id string) + // Get snapshot schedule for an instance + // (GET /instances/{id}/snapshot-schedule) + GetInstanceSnapshotSchedule(w http.ResponseWriter, r *http.Request, id string) + // Create or update snapshot schedule for an instance + // (PUT /instances/{id}/snapshot-schedule) + SetInstanceSnapshotSchedule(w http.ResponseWriter, r *http.Request, id string) // Create a snapshot for an instance // (POST /instances/{id}/snapshots) CreateInstanceSnapshot(w http.ResponseWriter, r *http.Request, id string) @@ -8763,6 +9247,24 @@ func (_ Unimplemented) RestoreInstance(w http.ResponseWriter, r *http.Request, i w.WriteHeader(http.StatusNotImplemented) } +// Delete snapshot schedule for an instance +// (DELETE /instances/{id}/snapshot-schedule) +func (_ Unimplemented) DeleteInstanceSnapshotSchedule(w http.ResponseWriter, r *http.Request, id string) { + w.WriteHeader(http.StatusNotImplemented) +} + +// Get snapshot schedule for an instance +// (GET /instances/{id}/snapshot-schedule) +func (_ Unimplemented) GetInstanceSnapshotSchedule(w http.ResponseWriter, r *http.Request, id string) { + w.WriteHeader(http.StatusNotImplemented) +} + +// Create or update snapshot schedule for an instance +// (PUT /instances/{id}/snapshot-schedule) +func (_ Unimplemented) SetInstanceSnapshotSchedule(w http.ResponseWriter, r *http.Request, id string) { + w.WriteHeader(http.StatusNotImplemented) +} + // Create a snapshot for an instance // (POST /instances/{id}/snapshots) func (_ Unimplemented) CreateInstanceSnapshot(w http.ResponseWriter, r *http.Request, id string) { @@ -9702,6 +10204,99 @@ func (siw *ServerInterfaceWrapper) RestoreInstance(w http.ResponseWriter, r *htt handler.ServeHTTP(w, r) } +// DeleteInstanceSnapshotSchedule operation middleware +func (siw *ServerInterfaceWrapper) DeleteInstanceSnapshotSchedule(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "id" ------------- + var id string + + err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.DeleteInstanceSnapshotSchedule(w, r, id) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + +// GetInstanceSnapshotSchedule operation middleware +func (siw *ServerInterfaceWrapper) GetInstanceSnapshotSchedule(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "id" ------------- + var id string + + err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.GetInstanceSnapshotSchedule(w, r, id) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + +// SetInstanceSnapshotSchedule operation middleware +func (siw *ServerInterfaceWrapper) SetInstanceSnapshotSchedule(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "id" ------------- + var id string + + err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.SetInstanceSnapshotSchedule(w, r, id) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // CreateInstanceSnapshot operation middleware func (siw *ServerInterfaceWrapper) CreateInstanceSnapshot(w http.ResponseWriter, r *http.Request) { @@ -10604,6 +11199,15 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Post(options.BaseURL+"/instances/{id}/restore", wrapper.RestoreInstance) }) + r.Group(func(r chi.Router) { + r.Delete(options.BaseURL+"/instances/{id}/snapshot-schedule", wrapper.DeleteInstanceSnapshotSchedule) + }) + r.Group(func(r chi.Router) { + r.Get(options.BaseURL+"/instances/{id}/snapshot-schedule", wrapper.GetInstanceSnapshotSchedule) + }) + r.Group(func(r chi.Router) { + r.Put(options.BaseURL+"/instances/{id}/snapshot-schedule", wrapper.SetInstanceSnapshotSchedule) + }) r.Group(func(r chi.Router) { r.Post(options.BaseURL+"/instances/{id}/snapshots", wrapper.CreateInstanceSnapshot) }) @@ -11792,6 +12396,120 @@ func (response RestoreInstance500JSONResponse) VisitRestoreInstanceResponse(w ht return json.NewEncoder(w).Encode(response) } +type DeleteInstanceSnapshotScheduleRequestObject struct { + Id string `json:"id"` +} + +type DeleteInstanceSnapshotScheduleResponseObject interface { + VisitDeleteInstanceSnapshotScheduleResponse(w http.ResponseWriter) error +} + +type DeleteInstanceSnapshotSchedule204Response struct { +} + +func (response DeleteInstanceSnapshotSchedule204Response) VisitDeleteInstanceSnapshotScheduleResponse(w http.ResponseWriter) error { + w.WriteHeader(204) + return nil +} + +type DeleteInstanceSnapshotSchedule404JSONResponse Error + +func (response DeleteInstanceSnapshotSchedule404JSONResponse) VisitDeleteInstanceSnapshotScheduleResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + +type DeleteInstanceSnapshotSchedule500JSONResponse Error + +func (response DeleteInstanceSnapshotSchedule500JSONResponse) VisitDeleteInstanceSnapshotScheduleResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + +type GetInstanceSnapshotScheduleRequestObject struct { + Id string `json:"id"` +} + +type GetInstanceSnapshotScheduleResponseObject interface { + VisitGetInstanceSnapshotScheduleResponse(w http.ResponseWriter) error +} + +type GetInstanceSnapshotSchedule200JSONResponse SnapshotSchedule + +func (response GetInstanceSnapshotSchedule200JSONResponse) VisitGetInstanceSnapshotScheduleResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type GetInstanceSnapshotSchedule404JSONResponse Error + +func (response GetInstanceSnapshotSchedule404JSONResponse) VisitGetInstanceSnapshotScheduleResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + +type GetInstanceSnapshotSchedule500JSONResponse Error + +func (response GetInstanceSnapshotSchedule500JSONResponse) VisitGetInstanceSnapshotScheduleResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + +type SetInstanceSnapshotScheduleRequestObject struct { + Id string `json:"id"` + Body *SetInstanceSnapshotScheduleJSONRequestBody +} + +type SetInstanceSnapshotScheduleResponseObject interface { + VisitSetInstanceSnapshotScheduleResponse(w http.ResponseWriter) error +} + +type SetInstanceSnapshotSchedule200JSONResponse SnapshotSchedule + +func (response SetInstanceSnapshotSchedule200JSONResponse) VisitSetInstanceSnapshotScheduleResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type SetInstanceSnapshotSchedule400JSONResponse Error + +func (response SetInstanceSnapshotSchedule400JSONResponse) VisitSetInstanceSnapshotScheduleResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + +type SetInstanceSnapshotSchedule404JSONResponse Error + +func (response SetInstanceSnapshotSchedule404JSONResponse) VisitSetInstanceSnapshotScheduleResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + +type SetInstanceSnapshotSchedule500JSONResponse Error + +func (response SetInstanceSnapshotSchedule500JSONResponse) VisitSetInstanceSnapshotScheduleResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + type CreateInstanceSnapshotRequestObject struct { Id string `json:"id"` Body *CreateInstanceSnapshotJSONRequestBody @@ -12754,6 +13472,15 @@ type StrictServerInterface interface { // Restore instance from standby // (POST /instances/{id}/restore) RestoreInstance(ctx context.Context, request RestoreInstanceRequestObject) (RestoreInstanceResponseObject, error) + // Delete snapshot schedule for an instance + // (DELETE /instances/{id}/snapshot-schedule) + DeleteInstanceSnapshotSchedule(ctx context.Context, request DeleteInstanceSnapshotScheduleRequestObject) (DeleteInstanceSnapshotScheduleResponseObject, error) + // Get snapshot schedule for an instance + // (GET /instances/{id}/snapshot-schedule) + GetInstanceSnapshotSchedule(ctx context.Context, request GetInstanceSnapshotScheduleRequestObject) (GetInstanceSnapshotScheduleResponseObject, error) + // Create or update snapshot schedule for an instance + // (PUT /instances/{id}/snapshot-schedule) + SetInstanceSnapshotSchedule(ctx context.Context, request SetInstanceSnapshotScheduleRequestObject) (SetInstanceSnapshotScheduleResponseObject, error) // Create a snapshot for an instance // (POST /instances/{id}/snapshots) CreateInstanceSnapshot(ctx context.Context, request CreateInstanceSnapshotRequestObject) (CreateInstanceSnapshotResponseObject, error) @@ -13584,6 +14311,91 @@ func (sh *strictHandler) RestoreInstance(w http.ResponseWriter, r *http.Request, } } +// DeleteInstanceSnapshotSchedule operation middleware +func (sh *strictHandler) DeleteInstanceSnapshotSchedule(w http.ResponseWriter, r *http.Request, id string) { + var request DeleteInstanceSnapshotScheduleRequestObject + + request.Id = id + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.DeleteInstanceSnapshotSchedule(ctx, request.(DeleteInstanceSnapshotScheduleRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "DeleteInstanceSnapshotSchedule") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(DeleteInstanceSnapshotScheduleResponseObject); ok { + if err := validResponse.VisitDeleteInstanceSnapshotScheduleResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + +// GetInstanceSnapshotSchedule operation middleware +func (sh *strictHandler) GetInstanceSnapshotSchedule(w http.ResponseWriter, r *http.Request, id string) { + var request GetInstanceSnapshotScheduleRequestObject + + request.Id = id + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.GetInstanceSnapshotSchedule(ctx, request.(GetInstanceSnapshotScheduleRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "GetInstanceSnapshotSchedule") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(GetInstanceSnapshotScheduleResponseObject); ok { + if err := validResponse.VisitGetInstanceSnapshotScheduleResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + +// SetInstanceSnapshotSchedule operation middleware +func (sh *strictHandler) SetInstanceSnapshotSchedule(w http.ResponseWriter, r *http.Request, id string) { + var request SetInstanceSnapshotScheduleRequestObject + + request.Id = id + + var body SetInstanceSnapshotScheduleJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.SetInstanceSnapshotSchedule(ctx, request.(SetInstanceSnapshotScheduleRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "SetInstanceSnapshotSchedule") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(SetInstanceSnapshotScheduleResponseObject); ok { + if err := validResponse.VisitSetInstanceSnapshotScheduleResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // CreateInstanceSnapshot operation middleware func (sh *strictHandler) CreateInstanceSnapshot(w http.ResponseWriter, r *http.Request, id string) { var request CreateInstanceSnapshotRequestObject @@ -14163,239 +14975,252 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9a3MbuZXoX0HxZitUQlLUw7LMram9Gsn2aMeydS1LuZuhLwV2gyRG3UAPgKZMu/w1", - "PyA/Mb/kFg6AfhFNtmRLsmJvbWVkNp4HB+eF8/jUCniccEaYkq3Bp5YMZiTG8OeBUjiYXfAojclb8kdK", - "pNI/J4InRChKoFHMU6ZGCVYz/a+QyEDQRFHOWoPWKVYzdD0jgqA5jILkjKdRiMYEQT8Stjot8gHHSURa", - "g9ZmzNRmiBVudVpqkeifpBKUTVufOy1BcMhZtDDTTHAaqdZggiNJOpVpT/TQCEuku3ShTzbemPOIYNb6", - "DCP+kVJBwtbgt+I23meN+fh3Eig9+cEc0wiPI3JE5jQgy2AIUiEIU6NQ0DkRy6A4NN+jBRrzlIXItENt", - "lkYRohPEOCMbJWCwOQ2phoRuoqduDZRIiQcyIaxpREPPCRweI/MZHR+h9ox8KE+y/XS836ofkuGYLA/6", - "Sxpj1tXA1cty40Pb4tivdn0jUx7H6WgqeJosj3z85uTkHMFHxNJ4TERxxP3tbDzKFJkSoQdMAjrCYSiI", - "lP79u4/FtfX7/f4Abw/6/V7ft8o5YSEXtSA1n/0g3eqHZMWQjUBqx18C6euL46PjA3TIRcIFhr5LM1UQ", - "uwie4r6KaFM+FR/+/5zSKFzG+rH+mYgRZVJhVoODx/ajBhefIDUjyPZDFyeoPeEChWScTqeUTTea4Lsm", - "WBFRJBxhtTwdLBXZNpQzpGhMpMJx0uq0JlzEulMrxIp09ZdGEwqC10ynWzSabPmqpeYkR7GsG901QZSh", - "mEYRlSTgLJTFOShTe7v1mylcGCIE91Co5/pnFBMp8ZSgtiabmnYzJBVWqURUogmmEQkbnZEPEcxmfudj", - "REPCFJ3Q8v026NTF42Bre8dLO2I8JaOQTi0nKg9/BL9rFNPjKASt/RvRF23RbB8wpSCT5fleAOmGSQSZ", - "EEE0jn/hdIngc8L0bdHz/Qnmbf2vzZxFb1r+vAnAPM2bf+60/khJSkYJl9SscIly2S8ajQDUCHr41wyf", - "Vp11AaOkwmL1/YAWX+EmmvU1gs2Zafq501J4urbLO92mSjuBNNopS1SglkQ+nxPmEZICzpT9UIbOKz5F", - "EWUE2Rb2LDRN1BP8FHEgiV8JDhn4ly+/XvctiJf5oWY0/a3TIiyNNTAjPi1Cc0awUGNSAmYNC7MD5aur", - "Bf9p6fpUeBWWZLSagpxSxkiIdEt7sU1LlEqQVJe2D7foiqrRnAjpvXOwrF+pQrZF7VARD64mNCKjGZYz", - "s2IchnBfcXRa2olHWiuJvzjRRNANCFKERIqjs18Otp/sITuBB4aSpyIwK1jeSaG3Ht60RQqLMY4iL27U", - "o9vNefQyhvgx4Cy7GHW8J8NAh5iG0rXsaerhO60klTPzF9BuvSrgfZoMaPSK9N/vPZs+BCJhtIRanckv", - "A75JzGGjacQ1TBcoZfSPtCRg99Cx1hUU0oyChiTsIAwfNMnGqeLdKWFEaDqFJoLHIG0VhGDUJr1pr4OG", - "Wi7saim4i7e7/X63P2yVxdhotztNUg0KrBQReoH/7zfc/XjQ/Xu/++x9/ueo133/1z/5EKCpZO6kQrvP", - "trv7HeQWWxTXqwtdJ8rfmvoXl++jOOaojzWduOlJHx4vCw5mryEProjoUb4Z0bHAYrHJppR9GERYEanK", - "O1/d9qvCAvaxAghsqsF0QzBUlB5A43bEr4kINAWOiEY82dFEmCrZQVjrzUC8kOaS/4kCzPRdMMIFF4iw", - "EF1TNUMY2pWhFS+6OKFdapba6rRi/OEVYVM1aw32dpbwXCN52/7Rff8X99PGf3lRXaQR8SD5W54qyqYI", - "PhuuPqMS5WugisRrT8RBN41AzIspOzbdtrKVYCHw4stP2G1k1UkbZa72qIPYI/m/mRMhaOi46uHJEWpH", - "9IpYdEciZWiY9vs7ATSAP4n9JeBxjFloftvooTcxVZqbpTmTNtagXvG4f2uRYMZBzogirjeUgbpGiMlh", - "GAgC+gmOVrLhVSD2AuswG3eZaf/CperGmOEpAW3SNkRjwa+IXihKeEQDSiS6IgstpCzQVA/anVNJ9fUh", - "bI7m2BgNekP2bsYlMU3cJ62IBITOCYp5cIWSCAdkxkERn+MoJbKDrmdaYtDEWBAc2Z+RIDGmbMhmepEy", - "4AkJtQ5hmsHW0CVh80sU4wRuKRYEriiKsSKC4oh+JCHipktMQqoZ1JARwGuUYH1lg4ALzX312RIczApQ", - "+LNEl0beuIThLynTWHlp7lVvyIon/6n15vzdz2/OXx+N3pw+f31wPPr1+f/on02n1uC3Ty1j38wEjZ8J", - "FkSgP32C/X420mlIRGvQOkjVjAv60RhbPndaGgZS4xdOaI8nhGHaC3jc6rT+Uvzn+8/vnTylpyJsrq+B", - "Z2GfvbKMYYUeinLkjHkSWQMRiHYYTLVAYV6enm9q5ppgKdVM8HQ6K18My9lvdCVCKq9GlI/GiW9NVF6h", - "4803SMsdKKL6gmZyxla/f/Lzphy29D+euH9s9NCRubWwfE1CuLDij5xp9NFCOKDM4ek5wlHEA2sCmWhd", - "aUKnqSBhr2J5g9F99JkwJRYJpz4drEKc8qbLNKrbzb/egBRtjinblPoYusHN4A54c2tN4DmbU8FZrLWx", - "ORZUs1lZviuv3xw9Hz1/fdEaaDoepoE1Kp6+efuuNWjt9Pv9lg9BNQatoYEvT88P4aTMtVFJlE5Hkn70", - "SAIH2f5QTGIujAZs+6D2rCwomHuL4HCGrZ2XPxvk2noJeOUOJaQSWrtRzMBljNl++bMPW2aLhIg5lT4z", - "2S/ZN3fyBbZuyH0ZtyURcyIypAUs7hXUjyDiadgtTNlpTagggcAa7Vqd1h8k1nL4/KNGnXztnn5+61Uj", - "+XONYImjhDKyQrL8RiS8ay6uIo7D7tZXFvAYUXrs5S2+Nh/K52txgmQo0eosWSNYeE1DNRuF/JrpJXvo", - "qv2CssYZcf2gd4Kjf/3jnxcnuZq09XKcWEq7tf3kCylthbbqob0mkGwjaeLfxnni38TFyb/+8U+3k4fd", - "hBFEbiXU2fN/bkYAkq1xPSw9UxprZhksf5sRNSOiwL0dsuifjD4M3ZHDvcJWSubR4pvmEqHmcyIivCgQ", - "Xrum1lYfqF9lVYIquKu2nyajV0h3XkOG9WiOyb+s6ujbfT+h9SzKs6afNa2wfKHJSrKFbG2f2D+3l5dU", - "s6IrmoxAah7haWayXfXafHZFEyuKQw9zjFFkCEGYgvA+5lz1huxvM8IQnB0cMPlAAqB5UmGFDk6PJbqm", - "UQQGHiAqy6xFC/Y5WTHNpdL/K1LWQeNUaWmdK4Ks3gSTpLAWaDwmKGXYPWdXZGe7wSpeWbBcEcFINDKy", - "sWwIGdMJ2U61wIGtTrBURBhqnyZleB39enKG2kcLhmMaoF/NqCc8TCOCztJE04ONMvQ6Q5YIMtcqBJuC", - "sZHaefkE8VR1+aSrBCFuiTEMlpnI7Fvr/OXpuX2tlxu9IXtLNGAJC0kIa3YcRyI1wwqFnP1Z31gSloct", - "zl8Buv8ud1qS4UTOuBqB5rdYR53ObPNT0/pGtoBOax4kaflIt6vH+Roe5DXw5lSoFEea1pbESe/7vPH8", - "8KgNxrGkqL5YupdhN1blh9WmBhMzMriBLAvVfruHkZQa2z0KqvySBcTpmZ+aLXbN+MfMLWSl3SdXNb9g", - "rjMzSBVEduyO29ktoHScwaQMK/x1wHMgC6p5rV09JFJRZtBJt0VWIpSofam1eYvHWn+/7KDLv5R+0Hff", - "qRZavrhGBhpAT5j+qTh+1Six1lzQXCmsHA6Wtz+PA1nrqITmW0gJzKTmrVrGSkgP/QJEHCkSJ5qSsSmi", - "EklDfEmIGL/+T8SNUOO6DplemjRuHhYcmdFI0imjbLqhxXzNmHAYGsvSJFWp0O3mVObQLKOOs95UN/DO", - "rI4YehynUnPkIEpDgi6dheeyLBcu23+WVUJrEFrScAxIQLMBZU9txqnS0+sNx1gFMw0nnirj92W3LssL", - "KFuZ1r2H2rVkL2W3OP+zjFyUgWrtDRXCrzdn32jALFiwT9aZAa2g4jdRXpEFHLkzR+Ilg2TREum3Fwoi", - "eTQnlu0WbZljHFwZVmJcL6wZ0xgkrQ1SX//KFfVa59YdhYZXY/CXVYVlVAITsN1sjjFW+jf230VGhfTm", - "zHwdrRhLAsAH1WOAQBy77BhdiYAFAjGNLBEKqSCBWhqesumQgQvIpf2lZ0e71Jdcyyi+S+hTdryyYEHb", - "MX1KR4sKJ+vEPhhGb43HVCkSdsqywRUhiVy/KS1eW8O1x7ouyLWgjpBZg1HYUDwjbMJFQGKrJHyZ4vi8", - "MJhXjbvZEMseGQa+hTVbfEI4SSJKQuP+Y84DzKzSnhPYWKsuv2FFazMeAOUpL3EUXaK2bbSBBNF7ke6s", - "GGc5sr87PHUokL1aX5x0NEZqKnA5UyoZ6f+RI32LL6uD2b7uhuvhNE+SaL8P+tXu7o49VWt0MwuuDFu2", - "r3m9GuqPxonf9Q9jPNZ30bmJNBHlD/MuuSX1irKw6QC/6ra11rlMMHKaxl0b6BJBumkyFRg8ZL+mee7W", - "z54AzXoKvsb53eflmEE1SKXiccHXEbUrHhq07MtRBtacR90QKwymzIb2VrPcZb/heGGGMrpYnSVmNB17", - "3H7oR0110ZRO8Xihyu8HW32fxvelb9BuLb5jqfO/NxokCUeKr/ZAphPk2jZxOAR+MlJ8NJ9Qz8gZW8vd", - "V6hEQcXZ3+q1eohuElBrTgAZJ5gZB1EDBBAaL06Kb3e9IesC+x2go2yCbNhsSAyyJQ7Ny0mbi8IiKHid", - "ofFiA2F0cdJD77LV/lkirbDMiQtImGGJxoQwlILpGbhh1/Di4gJSCUxTVbtb24mJXdiAJ0puv/XQL4uE", - "xNjaofRViLGiAXg6jWllP8COzEHZN2HMilawRlarVX7bb8mUSiUqXtuo/fbF4c7OzrOq/XL7Sbe/1d16", - "8m6rP+jr//97cwfvrx+e4RvroExbrO9Ykfocnh8fbVtjaXke9XEXP9v/8AGrZ3v0Wj77GI/F9PcdfC8B", - "HH5SdpQ7vaF2KonoOjKpscrn6lbwKKtxZbu1h9odOZzl/rOr2hpIvNMt7yIyxefzbD1ubx47UiWYa72m", - "C5tb1uQXCeid+S0pSHDWOTGgXjfMIyqvfhYEX4X8mnn4doynRI4MP/P7M6TSONmQD9a6IThXE2neTctW", - "z63dp7v7O3u7+/2+JyBjGeF5QEeB5kCNFvDm8BhFeEEEgj6oDQ9eIRpHfFxG9Cc7e/tP+8+2tpuuwzzx", - "NINDpni5XqhtIfJXF9znvpQWtb39dG9nZ6e/t7e922hV1l7caFHOtlwSSZ7uPN3d2t/ebQQFn0D/3AXI", - "VAX40Oe6oPUn89jYlQkJ6IQGCEJskO6A2jGwMJK9VpXv5BiHI2s88fMOhWkkV3pMmMlsS2Noi9NI0SQi", - "5hscSCNbNOz8CEbyeaNQxogYZfFDNxjJhhWt9RBwe8maoFJ4WAl0J1SCFJILT5RE4cDc0LV0Dk4zX9j7", - "Ojywe2iIDa+06tSNyJxERSQwrEsvNuaCoAxPzKGVdkXZHEc0HFGWpF6UqAXli1SALGoGRXjMU2WeGeHA", - "ipOA0zLoHhNNrpvpuS+4uFrr/qk58UikjOlh1lqFDsCQPrGmGuDiGNneLsKgIPRlz4Hm0dR+l+it6WEs", - "RPnPSaoQZYpr7ZSF40UHZrKWJIYEkYoDJbUGQztMU+nSL7eAsdS5f5j5ctp5T74v3YlxF/i6GraYEjWS", - "Cqu1EovGlHfQ/gyaN/Ym1x3XGlIawJ2R6/sAOrjbdzXadiXDyd1AfJUzWmZryBsBFxY0JD0Etwu8Ylx4", - "X+WmnSmeJCTM7D+9ITszVyX7SZoXFN3RwEHNCBWICzql5YnLBra79Gq7CSo6bLo1OhY7Lkuo8BHcN+ov", - "PZ4oIgwEXeRyMfzIHkKr07Kwb3ValhKVQeN+9EAkd7VcWuLL0/Ob+qYlgk9o5Nku+ELYr1Yzc15br3b7", - "Z92t/2M8MDW+gYhGmfGfiHlIepXkANC+Ged5eXp+WremLDMDKq5uaU+Zx4uHcmR+DQ4i9lHJvkpaDcah", - "v2Ys2SS57P3MJ8tOBI7JOJ1MiBjFHuPaC/0dmQbGtYkydPJzWZ7VcnNTrfm0dDigNk9wYAPrm0HfY5Cr", - "bKNTgOZ7/3G9JYYN14Xj6aMSto2NyOuh11kuDPTy9Fyi3EvJY6krH2+tv/zpbCFpgCMzoomupaxoYAPk", - "bCwhn+YdrSnSIyfHXtnQXQTUnk+TFK7h2dvu8ZuLzTgk805pTeBZNOMR0eveKFCLuQvKy537S0RiXmfp", - "MIghm16gAqyyG9wYSIX76oGO4gpHIxlxn7PGO/0RwUfUvnhhgqb0CjooKR2l/r0AhRJ+73lvjKZIddOe", - "wYRVk2npgnt1x3IKGWNeKWyvNKnvqvxCcGQy55TxOY/vdgfPr8oHza/W3l47iG/eY+cY3iB46/DkyAgM", - "AWcKU0YEionCNk9PwcUFxKFWp9XVPCrEJAZXu8l/rvZuqTHBF6Oxao24h0tpN+7EgFsTLv7WuCCEKMaM", - "TohUNly8NLOc4e0newOT1CIkk90ne71e76YxKs/zoJRGR7FpXPgL4So9Ofuyc7iDUJQme/nUOj1490tr", - "0NpMpdiMeICjTTmmbFD4d/bP/AP8Yf45pswbwtIoDwqdLOU/KT9pap5lfh/onTDrEqZxiYMCv/aJqUaf", - "Ac8GiJvzhgsrPNX6icG4L40LvnXmkDx9lSpkDCk6hDbIHkI/rraEOsEI2tg5U6ZolCdWWbaB3io1jlyZ", - "PWApc0BCWJYvIIrMXwFnc30rfMkDSgTcffui9wPr5TIKqQeT/2a1PeMkAVFV6+9baxMnyXq09QuKGf1r", - "mjTFhjZ7ONGDU/3bvLGVZ38z/e8//q88ffr71h+vLi7+Z/7yv49e0/+5iE7ffFEE1eqo9gcNTf9q0ejw", - "sFQKSW+KSidYBR6BasalqoGw/YIUN/6aPXQIit9gyLroFVVE4GiAhq2Ki/CwhdrkAw6U6YU4Q3ooG+mw", - "oTufGvOP7vzJ6Zafq2OENqRB2APJIplkOg55jCnbGLIhs2MhtxEJb/r6rxAFOFGpIPr0tAwbLdBY4CAP", - "Zcgn76BPOEk+bwwZaLjkgxJ6BwkWKkvD4WYApLCrMj4DtjkJXWC40ZCHLONLWVy4sdH0MiMI2OarHpd+", - "oHjVFy7KoTj7fV8EPXh96YOMqFQEHLMzzNZolLmjof1+iVTs9/f7awX8DIdWoB/chOUkmQ4pG9wlg8Aw", - "tSHc4KHWwJauaZO5I+iXd+9ONRj0f8+QGyiHRXbERskzPoDS2AhVJAvefxstn+nbnG7DDRkjGXSLGkQN", - "PTfuoe9enSFFROwc9tuBBueEBnp/8PxPpUw1KlKMDg5Pnm/0GmT5BNhm619xju+yHVaDO6zRrM4WmGG8", - "hm8HHR+Be669obkAB241L7hAkSEw+b0eoHNJyr6ucFTmVd+cZLTILW+GAwxbG27EpEopBuhtJjfibCmZ", - "o2WODG7I/F7CsPbhxfj8LI1e8csFbyarF1nSBh4+WGVO4prj1pOC1dffA3G489avu2DTvNndLhpD9WR+", - "1MjP/s6llZ2b6qg3TdBQjqEsxN9mORqaJ1e4iyQFy/raB6pGtY/wSH+2T+5OK7k4QTMs2Z8VfKzoJls7", - "Txtly9SzNn2+Lj5c84lZUnarXEBm9uxqQlOvaBQZbwZJpwxH6Blqnx2//PX41asN1EVv3pxUj2JVD9/5", - "NMjV4FD75ek5RLtgOXIvQPVOjzh3HCYfqFRyOV610UPq6twQv5TyN3gDgDe+YlIH9/q8tI37SNfwkG59", - "316qiJXJHb40Q4MVdu8oQUMtcfUlNyjTWfPz1021cCfLKcX++OhDUSZwPte3zm3QaVGPv+mB1CSQhOj4", - "NE9xmBul3PCVPT3b7m3t7fe2+v3eVr+JiS7GwYq5Tw4Om0/e3zaGiAEeD4JwQCZfYCK0iG2ENxxd44VE", - "QydeD1tGni8I8oVra0XwRs+vyykkbpcxoipQrMsJcZMcEM2SO3xpRP2qRMdn5RTHjYW8J3//omzIpClr", - "t74PttfoJtZvggKeRqEWpMb66hq9jIRWfZRE5dmj4bafsyvGr1l568YIqgnAHykRC3RxclIymQsysclx", - "G2wcfCZqzoEnNzqG7TWy9trV3DLPwn3kVqiS3QK7++qZFIo2O+eDaTC0ge0uFz+97+aUmaPReLJiTxWr", - "S0jmozT1SVX6k4u8OD8/PiohB8Z7W/v9/Wfd/fHWXnc37G918dbOXnf7Ce5PdoKnOzXp6Zv7zdzeFaZ8", - "m+sjnQDwYME0gWzhQN+3zJdlnCqU+bnpi3yoxVNUkINNXA8YFY4ZVZDDkbKpHgZ0fCsmmwBNk2aSMqog", - "IwDko6FMbxmMKXoQ6700QC+hLXzCMcQbuUVo5ahsR8DhwthRNWFwUyfwr9VLPpulSstt0EfOUoX0v2Db", - "GgxWXVk9hKExA/SaQx/hnEwZr+o9pjk4by03r+pIbetW5NxPYTJLMAfoRUYkMzJryWpbEvunod3WMxq8", - "vjdKvnf2xFsaW/KTK7iVdVoGoq1OywEK3M+WHdHsurwxFkVU9D0wEBwBCc0dfVJFI5vkAHZCpaKB0Rox", - "HG7dTbYJvUg4MiJA3XOh8R6xYkLWyRGKixPUhnDGvyKrVOp/bWRPi8Vbubv9bPfZ3tPtZ3uNghbyBa4n", - "8Ifg27S8uLXUPkjSkav8UbP1w9Nz4H2ar8o0NlYCu/eCj2gieKClVcpQXkokn/xZ71kxViPk6TgqWJ1s", - "YBcEBDSp+1LzPvYHjeZ0MmF/fAyutn8XNN76sCe3x17lLpvILwkfFy2lS2ojGXdNEka/Oz0glJC1ESdv", - "iYQdoDOiEOBPF+EAmHTmkmRRzsWlWIh7EWt3Z2dn/+mT7UZ4ZVdXuDgj0F+XV3liV1C4YtAStd+enaHN", - "AsKZMZ2fJuSHYFaA898zZPMx90sunFp32vFhSY28lGONHXse14L8wgpBdlMW6OBZlQlIS7fcC+2dnf7T", - "3Sf7T5pdY6uxjcSH1RTG5eQw4LFpTIon3wbr+ruDU6RHFxMclDWUre2d3Sd7T/dvtCp1o1VBCh6TOuMG", - "C9t/uvdkd2d7q1nolM+CboMCSxe2TLs8l86DFJ7T8IBimfR26riFT/A0CPaWBBGm8UHgvF8q3MekyBgJ", - "0yw/hCaMwRoJlhhXg76NVLRK2R4jGnCBUpYlZuqtN4fezrpZT6YNP1hPxpdl6AgzDS7r428yMd4Cdokg", - "c8pT+RUG4ooEGpkmEefiRn3r3IneEplGypggqUQXJ38GIqKRC0lFkrKrvEW/FZEQt9zcjS5wCSf8WF0H", - "rEan0eToV224U3NNO6vcYEvXvzbgKNSkKmXrn64PcRSkkHsMZ+epdwWhAzxV8NC+ME4eUcQ5Q8EMsymB", - "XO4m0yGbIoxmPAp7Lf9TSRSOJt4nDH6NIm5SJVwRkti0XGYRupuWWeicoPZLnheUM6hUSa/7JDZUxSZe", - "KmPjk7imOKf0OQ5mAUoanljxQhS/6VLS5iM+laAFKnBf6VWTxyRYGK8UzEyauXlslMdy5NW25vaeJVao", - "t4+FGtbJJ1ajtTKG4hkkcSC4lIhEdAopzS5Oystc5X8YU0ZjTWfXP0eXF9sAdWXCmfTlRQGeJhtno/Qx", - "RI9j15ewRMBh8N/0Ph2Yh3zrio9izFJI1FVAZPIhocKgR7PH8RmXapRFk9xwsVKNIAlTKkgecub45Qz8", - "9xeGxEEbL190pO024LJeE7fqvYRV/qHqFlhPU70Q9UOrk+GgD42X42lWhvDkMUHVAJCbRHzlWXuohFFp", - "IdgItRlXJbJUyDyz0eShyq+j6nnqCrq+2u2fNQ3GWh17dYrV7JhNuCe14w0M/taj3fkuJETEFNKQoZAw", - "SkKnPGaWf2vbAh/5SBIUpsRCzgikAluAY3O9IYUjc0YxyqYVWl+dsIkZ3qxhdY4mmNc2bPLkKP2e1e9E", - "CrAyTgIS4dzHupHHA5Ujv6V4eWBBpmmEBaoGHK5YslzEEWVXTUaXi3jMIxog3aH6nDPhUcSvR/qT/An2", - "stFod7rDKHcRrDzPmMVZB1FzIJV58y38pHe5UXFPB9PLpum/CRW7m7zgev2GXtCI2Ji8c0Y/FBC9nMRk", - "d7tfF7lQM2gpZmE5nvOmlNuirO/Gu1DLg6zogcc/zXgAVV4lyobI0n59uwUXs1VxGsumGNR2j8IuSUwZ", - "roVkLY0sIc283KruD241m5IE5dl395883WuYLeeLbJ0rahp/gWVzHq+waNac1EkTs9n+k/1nz3Z2nzzb", - "vpGBynnK1JxPnbdM8XwqtU0qRrMnffi/Gy3K+Mr4l1TjL1NeUKlOya0X9HnF1c2jpGuePWrTdEfFk3Tv", - "LGULaDMb4wpp6aAkchVKcbXJZEJAqRwZuHXzxVS86xutIcAJDqhaeAwm+NpkbM+aVKJ9m1jTyov1gNSO", - "bRM2aMol03Hu0Nl2k6O/GNN6BRf2Gyfdkum4zoz/pjqrMeLnNqDiE1GDF5q8LsCyuSDbzzWWJa8O/XcA", - "KZfzUmtV/yHTonlNaIfrWVno3DPSF7HuLwFdPP7KcRbMviUhuQrxVSy0/greSIf2cGRflcn1XrkV+mAZ", - "4O16jcbFdHgr8w2WcuflXPfm8zYrErfcz3Cwm89XcAG9ScdqZjDAR7sGC/J87E4JJWqwSXGxPiH0HeT3", - "MT4Ft8rwY90R7iXJj/35ThL7LB3HWcGP3Z+Rm4SjVTHIh1kzZw9M8ALkhloh/+nObr+/s92/VRDy10oU", - "Xhinztuv0M8q6qVnpeIImW/fcja5a0FNvSkHJqkEwfEAPGUSHBAUkQmE6GRZPNfqa0tTr168fQCzTt2Z", - "y407KFdU0urQFn0ZZ8BU3Dg2fttto+Ve4Mp+/MXvy8teEQjkDqtc08hVvS/5Ju51+zvd/t67rZ3Bk73B", - "1tZdRC1nQKpzz3j6cev6abSNJ7vR/uLpH1uzp9PteMfrCn4HOekrJd4qKertHhIiqmkCq+k1JYkoI12Z", - "uTStdy5dQQvMK8Ha+38zzdLsYCUjOCtvssgPsMqBU62WdR9BK3b1K9Xj6vKPj1Yv+1Y+QtWF+BGsuhTA", - "p2aLgSwaW1+asiFlDfnOeaFhY86z0m9tHe/xOfTC1faecg3EffhcIoylG/Z+Bcde5moe8XzKBVWzeDV7", - "yJplAeDw0PlRqrAcJNFDx1MGSUGLP2d27WLdXt251WlFH3fLd8b+3jxcxgY8Zwhoj7ooBjSw+0LO2dVQ", - "gCa52CjM0zMWBADx01Z36xm8vkYfd3/qd5/10N8Kr8AdA60i+LZc69Kv/SYwzAglPNyaV9GtZzd6InXw", - "XIVBv1q+VMeIbSi0xfE8I6PjFc4ltnTA+eelM65EjNxRDZhVUm7zdJUVhcE4FBezg2YBoF83V6XtuTaT", - "8N3B6p2l1nUBy1VXgmIW14Pu303WVjTqDTZ/+uv/7r7/y5/8mbtLtFwS0Q3JBOwrV2TRNSXJNN/olbNd", - "QTB1Syps810rgmMQZIIrYgSnGH8orvdJP3v5XLzG8dIWwDAVU5b9e+2G/vqnerNOAYznSbhckqq2pNxX", - "zRumOEphdtSOiZi65JzuMRFqNWql+IosJCokJ7F+kI5h/VlmXYpVwS4NA+tBLbsxhRxPcsg0ocRBQBJF", - "wp5N0kBhLYLDjarWxrNJUpzzj75rGDJDWWfMSg6ET94KdIMWI9ddM0PY1aiz+2TPlgMtQnJr6YR8Z2bi", - "eerK52goewSRV1SCs5rzySg0Rm0SJ2rhUoC5V/ONm8UXHWQDeg1lXzm5Qv/Z10gFdb4y99N3WLypGP7l", - "FrQ28Gvp/GsTrvif3Y6qcdzmTtqCFOW440qafam69a9yMU+ZGsHj8vILmv5mHq5tsqNpWs36uBkztWlT", - "q/mc5UKoN7fSVSG/ZS4Wqgud1r/Ar/TrLOyssJL6szGxf8sZjVYA6FSD5npGBCkcBHTI80PdEGT2GbmB", - "C6ZJgJQQ0a1WLDFJdQWFd2kLIANYDYLM1WBZtF4d/3yCP2QzgFqG5ZLxAvaRZwKBCvUbPfTWVa6gEzcE", - "LKPsDLPlD2YuY1GT2pvLh1HEquV9m/bei2dp1QrqV3e3KsiZz1FCzfc+biZJkAqqFmeaDNksDVC59yA1", - "aAj0CTYBP+eTQwatz5/huX3ieXV7SRgRNEAHp8e2bi8D+QtdnKCITkiwCCJiEyAtxQyC6PHm8LhrMrdl", - "BeH19FQBQFzls4PTYyikJIxU2+r3tnt9QLGEMJzQ1qC109uCslIaDLDFTUi4CX9ahxrjS0s5Ow4td/7Z", - "NNG9BI6JgvL8v3kcUxQRJoGnBFkITwvCaIKpsNJoEoG7jNE1qe4LIduOwA8Ml+gYgOOmlg+pFvbxkCRv", - "7LG+1+hgnFdhi9v9vhH+mbLsAOcFdjZ/t07G+byNpAwAjyd+eUnadJKOBfnnTmu3v3Wj9aytieOb9pxh", - "WwCawDKf3BAIt5r0mBmPBmRi46xdv3jPAIWKN+y39/q8ZBrHWCwcuHJYJVzWiWhEIgxVOUz22N/5uIes", - "JQ5SNskZT6MQjQky7hok1GQUI4VFb/oRYRHM6JwMmeUepr4RFpDSLkaaaxhhunw1zNTm9DNH2p95uKhA", - "NxtuUw8H0lYZwNXUMJKMIMJ9VJcaOjNjJ5QxKDEjic2jk+VIXX48gJpgMuDeYmiEYabyElOmGNgVgQi+", - "Cf3gHbBRKKomeHAsBGpPZikStzf8PliQ8cfvvniUfUMWvGUmp3UEW4w9kwScWwAWYxxF3lilacTHOLI1", - "066IR3B6CS0sUIrJkRzLZTwkJtFNslAzzszf6ThlKjV/jwW/lkRoxmwT3llYu9LTBnWheCWNIemcSaer", - "59w0S9z8dEUWn3tDdhDGLlWytPUuI8ltMbmsOLl7Rza460/JVGPGPrTFZU0hp2LtG7NMnqokVdaEJ4my", - "WfqgOZRGkjMSDpni6JMwlTAXnzc/5TN+Boma4FDjSaGJ2dLmJxp+rlu1HGG9+xE09egkBAAwbGnuMmzp", - "v6cCa4k6lTNQsCUo1dPikbaz+BEtrWxUIRxghhKepLbuv6YnUCOvNAZkPMVRhBRcJddXy0BwkjX7se50", - "vvId1pfOOD9VrhEU8ihcpv7uvv8+SRII4lO7//vszWsErEqfgWmWGzMARpRpLorCFORLmL03ZM9xMLO1", - "4SG/wrBFw2Erk3nDDVhrKu1jf7cLgtdPemk/mWk6NPyp19NDGZlugH77ZEYZ6LuUxCPFrwgbtj53UOHD", - "lKpZOs6+vfcDtM4l6axECFDb0P4Nl68aQqNyNmj4BmYh4pbWRguEUU6Bitr9mDIsVibb9oDeQlArmHgq", - "i8D4NASz3LA1GDrD3LDVGbYIm8Nv1no3bH32Q8Amh68P5jf5xm2zHIn2+v2N9b7CFr4eEbrUUF+/z0vS", - "1/ZXEzys0LUseJjNuUwk+gRN5ngjbt2D5PMzDl0u0h8i3hoRz+rTBeEN+hf5gEHfiJgXh4oEphXwyElg", - "K7UTgxaQigc0DufZbxQO6iS4HHmL6kdVyVxWK3brblkAS4wc/u3eA/7BvHn1RZj32X3NiyNTJ9zVIntc", - "6AiH5RCx49eIXxL1LWBc/75IqSsS+4D4+1jw5yWxcl8OtAo12yRz9wrij18C3zZpRzGNta56BmvqnhGm", - "0HP4tWf/6zQeyMZ1GfHp5QAZEEZ8iiLK7BtR4Q1DM0ULS+hk3NuyftbbzQWPtw3//Nc//gmLomz6r3/8", - "U0vT5i+47psmog+STV3OCBZqTLC6HKBfCUm6OKJz4jYD6WDInIgF2umDmJkI+OSpbyOHbMjeEpUKVnhL", - "M3Hc0g5oy63q/VCWEmndA3VDOrFBZsbs6VHh3V02oLzXG91Z9qUwOyhsQHNFhwMQNUBNxi2rf7X81jOz", - "55L9rGrBXbLpr6cvinxQBnu7ZoE3JDAAYt+9gw9206h9dvZ8o4dAxzBYAYGEIDHnw1jhufeDJq2nSYai", - "lAkKQNnQpkJpw1r775Ft08wAbEf8nizAdbUa603AxuRBBAkdvH7oCk3MwX64OdOwzz575JzC6w20t99v", - "cQrnotJIEf565+xwbxnm5ksBZA+hAqO2LTmflRk5PTx2+aw3Hgzp74Vr6J3aLLAZ60DcFDe5N7XskLNJ", - "RAOFum4tkIg0JpmqVkaQx0IO3tpVI+z2VU3ZUeRvm6UI1FpOlwWj5izv7rlHZdKbsJE8rUiOaz84yTrU", - "OaIy4LpvAVu6AU5skRUjvmT3tIhF6wxSR/B7xnJWikuWPB8fuQt5f6YpO3XKqrzhHojiUYUgPiAhrBSO", - "KCTieUzYfJ6doguAW2G5+rZQs39/UtB9W7F8aP6YzFhhBWyaCs6ycuJ16GULjt/hQdsZPBs/I8Ldapc3", - "DXadbct0RcGMBFdmQ/AgvVr3PTZNmqm+ZrzvSfM1ld5vILFYkP8QURoouzmsVim4x7YYxt3ptzDDjdTb", - "r/fOaxHMA2RwNhk7i7WpM4HlggUb39VT771wMwPsR8nMTtMoci8ecyJUXu++yAM2P4Fb0nrZ3t22lezg", - "/O2rLmEBBz+0zIfKL0S5MtRfV8I3B2a28gNNmuiEACqHGPUC9Becv3EXRFlNw//YfmGrGv7H9gtT1/A/", - "dg5MZcONO0OW/n2R5vuWuB8x8mmBm5aBBqTJFIteJ6FmrRoKqa79dyWnmk3fSFLN4PpDWG0irBbBtVJe", - "tUdxpxKrLZf/ME8yGbL5oA2fnH/idyap3q+Vz2KkyzlGZfnZwyaV5iIvUU8ZSiV5hA6UNMO4IttoaK7O", - "L+RK9uFQ9/ioA4DsaNAdH+UBIvdkvHbruHfh1s57/5brg3hMpylPZTH2JMYqmBFpg5UiUibAj03sztlz", - "reD9DWNp/z5Zx73L1T/w/o4k/uqBGuJtXqDWyfyuVVOZ37bXMr8t+Gli1966QqI2+c5GjVOhq7bRFI1L", - "VaGXnR196/LpIuhcKyq5uoBAgxgM2X9p/eM3RXD8/icXJJP2+9t78Dth8/c/uTgZduJQhTAlqE1KdfD6", - "CJ79phC9Dnmr8pC86jpMElNAPZdW5d9OQcpfPptrSA4Lf2hIjTSkArhWa0hZodq7VJHKmZXuXUdy+OYD", - "uE2t8UNLug8tSaaTCQ0oYSrP8b/kJGZLhDzC2DJm34cKzh0lRttYS8qrR68WQPMspPfu2JNNfv/KkUt4", - "+jh95LmJigmdOpIzw3p95FvDh/79Euf710MeM4oZgX8ZdImWKX2lYyD/YJwqcErMM4SA16cr/5+N2EN5", - "xRZbmFeaHIYgAEMKfzXTArAv32E5haEvZ2HCI80hZGfIIDGq/mxi+TevyMJkKKQ8r0Sa7dRmJfTFXpUT", - "PD7oNfr6MpY/e2UjGeuer7HJdPmAMtaDkY57kbSObfSDVbqziwEK5ZhkN5lnwX30I2XTjUflgWqIVba3", - "Qj4jj6i1ObFFVPwq0AsurpoSBU/i90dAG4o7/Aa1L708yJ/08EoYqCfm/mikuXe6sZTN/yGd1mmVkgRR", - "GmrS4UiIY74TweOR/dHkndS3wlbEBKUusKM+NJHRs9+Div2aK0TjJCJa7iEh6hps0qdphSWXvJnKQu2L", - "mxFBfW2KIQQmfZc0VtUOshVC4DnCHVgbXiaXj8tLNSM+XZ82IJvcxch78gYMmUkuTVwm6kuUEVmkOJIk", - "IoFC1zMazCCHgP7NlNyB8H6cJJdZ0qCNAXoJN7WYOwkmb0sitOgYcCZ5RExqgHkcXw6Wc1xenJxAJ5M+", - "wGSzvBwgl9cyYxBStyrmBNC7iLBU6LXNdNDWmCR4FJkTvdRydmF/GzZbQJ7Uach8mQMYubYD0gm6LCQR", - "uKzJIuAI6is+lQ8lynbqU/GZvUCheQ04g5sEyhL4Tdc08ucP2Op7Syc0zGVglnHHqQyWFvOKT7M0gCVU", - "xknSFH3tMgGL53G8AodRu1A9R6qQp+qvUoVECOhssbsOuVHbljNHCl9pRGW2nLKrPwTo532gMXm5vKDS", - "RLVQZ8H8ax7HrU7LrsdTV/7Lc0JUB1x+SNAnU0j88MOWcJOUDmViX8jpUOEctvZevchtSwp+9xYtC6jw", - "e9BLyy8A+Sooc6IKnC3P67Q8qthwU22yKouZojK+O5KVq6y/JeVns7O8fM2/oYpq9lqtMXrPSmoGYp9m", - "Vqqn9uDaaVZd64eGmmmoXKAwNdNVCix+t2pnRlBQykqapxVPb6t7Zmk0MzBDLXW28skzp3mbn9yfx7cQ", - "F74RStiprcNWl7At3/S3QHJr6jp/i08GTk6ybLUgIDwgCXYVph/sbUGrexmV+ybIsLlwGTUu0hwlMJPU", - "FY//QYxLZkBjKb0tMXbC55ItsECeKesmEa6jy1ZOrSXAlRqH/2ZPpzUVHD9bSviQhC9Xju6N2B1n5M0Q", - "PFui9ztXUQMuhPEGBv/ixxSNXnCOKOja7QSnknQyCtFxrkgXJycbdVRCqJU0QqhHTCEq1VpjT6mPN3Mi", - "BA1d3ZHDkyNbpYRKJFLWQ29iCsVArghJIMsw5ak0dTN7xRKSdcVIsxqRhCmxSDhlau0q8qZ3s5jPt6q7", - "cM900ubj+O7taPD29/iIFNAOLa7YDazWIpWpnOp9l3TvdJSZYilQLX3MUz36UolLNKERkQupSGweKSdp", - "BJcIMjbZhN62n3FH7iCqJNL3oQPumwkRMYUCzXLIxmSixbCECD03VJ6mESm8t/ie8s4UzqjmqSF938Zb", - "HlS9hOcrrOqgVi54iZPEFbz0vRdlNTpvvaQX8DiH5CIe84gGKKLsSqJ2RK+M0oHmEkX6j42Vr3sj6Pe1", - "05Xf/mZpSB+zCfdmdDU4myHz9+HBViZrznvh0ZG1l6R4WRz9gYP2kzW5lq4JgiOo65xFTqBU0Yh+NKRO", - "D0KlooEpg5f77UIFL+u6O2QnRAndBguCAh5FJFDOuLKZCB5sDtN+fydIKIS47RBYHBC8+s8xzHh4eg7t", - "TJWxzpDpf8DA7w5OEdUwnWBrIygslBF1zcUVOt58s8bf4QzA9G/8YGg2uNKB1nvgP56yb+4WX3uHZM0V", - "5ckqBYgn3/2LtpXgflgLHqe1AOKSst20pwIHIBTLWapCfs38lgFT9FpufjJ/HK+LblM4mF246v/fhrRr", - "C4Cvm8Zt8FFcSrunkJiM0w/yQGFrtD/SDH0acG4LIMQU4/T8XOBAfY/Y/fWN8kU4foNPkxaiLpv7N3O3", - "7pvz2TW4oO0iPB7LNTeY5nYCVYiL1qcsQn2tbhaYiEjI7pWLlgFOcEDVooNw5Apk24p3mQ2pm7HcsSD4", - "SnPa3pC9zWLjbcU9rV11nGqFQiqvzAhWe+qhN3MiZDrOFoeAMBk9D4Bva2QHOApMcWkymZBA0TkxVZ9l", - "jfaVLeUuM63nk3gO2n20oHtsKocfJ+D0crSQFYzbNEe9KUgQYRoXVY8qcIBAguEfjANjPShniLJJZI2g", - "geBSIjtUl0R0SseRNenJHno3I0jimAxZEmHGCBTUhtdpvfRuIoiUKXg7wABQysJgVAflkUGJ4MoaEyLO", - "hTT6v8bwixMkFUlWoNlbM7IpQn5HmUjM4HamB2IrlTWYWfwob0CtD8RgigG4xqM0UvIBnoTNgn4Um252", - "8d8JOp0SoW8FNkTWGNDNtXbgNJe+5A9cm4brLGvVLA1XNmrB56/gD7cysmPkGo5A/LyJnd4z+RWtDf6x", - "n27mo/ur7tRw7rIvqH8R9tMX7vJ7yW58VnDBa5q8K8fwx5ZHq7Dy0lUtubGuT+jT2G/1Lv1I1yX0ySa/", - "74Q+Z15XxkeWVhSXnFPrMvl8e4jQv98YivvO5PO4cUvrD3IJdPWUqEG+i28CA+8m0cUDxxDdItHFN+XV", - "DokKHi666JvyZ7d+2Zk/+49UFnfpxm7yWUDYfp0bu6F69rlppaJ0Yds0U5PsiN+TBG9fKG4gvzuw/9D6", - "G6gMBWD5TXYmgk5ahCdxohbOBM0n4GyX54aW9CO47PrCg7OXpruLyr3FI8zXQw+Hp7VPMD9S9t7bK09e", - "1+T46PHn6S3euRJj2dRcp4tFMKNzUm90L99gC6JEkG7CE3hcCQ3ALDwcL1NY9KYfkR2+N2TvZsT9C1GX", - "E4iEKKSCBCpaIMoUB4pg5vizRIJrTQC+c7HwGdOLN/eF4PGB3c0afmjvlDWG5b698aIbYoW7c0dtVpjQ", - "vuCd+gR/oHEaA8FDlKGXP6M2+aCESVGDJlrzQXSSgZR8CAgJJeDkRnHBW/0ayyb9SEbTcZNVrkg29MYm", - "c0JBKhWP3dkfH6E2ThXvTgnTZ6FF/QlIsongcxqaWgc5UOc8MlDdqgHoTe2uWqiwQSC5cmEW9yAyTBOG", - "NP1IkzJZML7OrUFrTBmGxa1N61O+U8btXs+HKTi/5nfHYU7rBwuzml/bKTsaE7WS44CoOEeRlug3frC5", - "x8zmiu5LjqeVuF2zbPTNPJoaOhrdRSb6zNvtfs3WF9+OEw6Vj9L/xprO55lCWmc2/7ZQsH9//OG+zeUX", - "j9hp8yVxynfBVA4D6BF9CPOKBzhCIZmTiCeQqN60bXVaqYhag9ZMqWSwuRnpdjMu1WC/v99vfX7/+f8H", - "AAD//xqeUJdsRgEA", + "H4sIAAAAAAAC/+y963IbOZIw+ioIntkYaYakqItlmRMde9SS7da2ZetYtubsNP1RYBVIolUFVAMoSrTD", + "f+cB5hHnSb5AAqgbUWRRtiRr7I2NaZmFSyKRSGQm8vKpFfA44YwwJVv9Ty0ZTEmM4c9DpXAwveBRGpO3", + "5I+USKV/TgRPiFCUQKOYp0wNE6ym+l8hkYGgiaKctfqtM6ym6HpKBEEzGAXJKU+jEI0Ign4kbLVb5AbH", + "SURa/dZWzNRWiBVutVtqnuifpBKUTVqf2y1BcMhZNDfTjHEaqVZ/jCNJ2pVpT/XQCEuku3SgTzbeiPOI", + "YNb6DCP+kVJBwlb/t+IyPmSN+eh3Eig9+eEM0wiPInJMZjQgi2gIUiEIU8NQ0BkRi6g4Mt+jORrxlIXI", + "tEMbLI0iRMeIcUY2S8hgMxpSjQndRE/d6iuREg9mQoBpSEPPDhydIPMZnRyjjSm5KU+y83R00KofkuGY", + "LA76Sxpj1tHI1WC58aFtcexXe76RKY/jdDgRPE0WRz55c3r6HsFHxNJ4RERxxIOdbDzKFJkQoQdMAjrE", + "YSiIlP71u49F2Hq9Xq+Pd/q9Xrfng3JGWMhFLUrNZz9Kt3shWTJkI5Ta8RdQ+vri5PjkEB1xkXCBoe/C", + "TBXCLqKnuK4i2ZR3xUf/P6c0ChepfqR/JmJImVSY1dDgif2o0cXHSE0Jsv3QxSnaGHOBQjJKJxPKJptN", + "6F0zrIgoEg6xWpwOQEW2DeUMKRoTqXCctNqtMRex7tQKsSId/aXRhILgFdPpFo0mWzxqqdnJYSzrRndN", + "EGUoplFEJQk4C2VxDsrU/l79YgoHhgjBPRzquf4ZxURKPCFoQ7NNzbsZkgqrVCIq0RjTiISN9shHCGYx", + "v/MRoiFhio5p+XwbcurgUbC9s+vlHTGekGFIJ/YmKg9/DL9rEtPjKASt/QvRB23ebB0wpSDjxfleAOuG", + "SQQZE0E0jX/hdIngM8L0adHz/Qnmbf0/W/kVvWXv5y1A5lne/HO79UdKUjJMuKQGwgXOZb9oMgJUI+jh", + "hxk+LdvrAkVJhcXy8wEtvsJJNPA1ws25afq53VJ4srLLO92myjuBNdopS1yglkU+nxHmEZICzpT9UMbO", + "Kz5BEWUE2RZ2LzRP1BP8FHFgiV8JDxn6Fw+/hvsWzMv8UDOa/tZuEZbGGpkRnxSxOSVYqBEpIbPmCrMD", + "5dDVov+sdHwqdxWWZLicg5xRxkiIdEt7sE1LlEqQVBeWD6foiqrhjAjpPXMA1q9UIduidqiIB1djGpHh", + "FMupgRiHIZxXHJ2VVuKR1kriL040E3QDghQhkeLo/JfDnSf7yE7gwaHkqQgMBIsrKfTWw5u2SGExwlHk", + "pY16clv/jl6kED8FnGcHo+7uySjQEabhdC27m3r4ditJ5dT8BbxbQwV3n2YDmrwi/fcHz6KPgEkYLaFW", + "Z/LLgG8Ss9loEnGN0zlKGf0jLQnYXXSidQWF9EVBQxK2EYYPmmXjVPHOhDAiNJ9CY8FjkLYKQjDaIN1J", + "t40GWi7saCm4g3c6vV6nN2iVxdhorzNJUo0KrBQRGsD/8xvufDzs/KPXefYh/3PY7Xz46598BNBUMndS", + "oV3nhjv7beSALYrrVUBXifK35v5F8H0cx2z1ieYT6+700cmi4GDWGvLgiogu5VsRHQks5ltsQtlNP8KK", + "SFVe+fK2XxUXsI4lSGATjaY10VBReoCMNyJ+TUSgOXBENOHJtmbCVMk2wlpvBuaF9C35NxRgps+CES64", + "QISF6JqqKcLQroyteN7BCe1QA2qr3YrxzSvCJmra6u/vLtC5JvIN+0fnw1/cT5v/7SV1kUbEQ+Rveaoo", + "myD4bG71KZUoh4EqEq/cEYfdNAIxL6bsxHTbziDBQuD5l++wW8iynTbKXO1WB7FH8n8zI0LQ0N2qR6fH", + "aCOiV8SSOxIpQ4O019sNoAH8SewvAY9jzELz22YXvYmp0rdZml/SxhrULW73by0STDnIGVHE9YIyVNcI", + "MTkOA0FAP8HR0mt4GYq9yDrKxl28tH/hUnVizPCEgDZpG6KR4FdEA4oSHtGAEomuyFwLKXM00YN2ZlRS", + "fXwIm6EZNkaD7oC9m3JJTBP3SSsiAaEzgmIeXKEkwgGZclDEZzhKiWyj66mWGDQzFgRH9mckSIwpG7Cp", + "BlIGPCGh1iFMM1gauiRsdolinMApxYLAEUUxVkRQHNGPJETcdIlJSPUFNWAE6BolWB/ZIOBC3756bwkO", + "pgUs/FmiSyNvXMLwl5Rpqrw056o7YMWd/9R68/7dz2/evz4evjl7/vrwZPjr8//VP5tOrf5vn1rGvpkJ", + "Gj8TLIhAf/oE6/1spNOQiFa/dZiqKRf0ozG2fG63NA6kpi+c0C5PCMO0G/C41W79pfjPD58/OHlKT0XY", + "TB8DD2CfvbKMuQo9HOXYGfMksgYiEO0wmGqBw7w8e7+lL9cES6mmgqeTaflg2Jt9rSMRUnk1pHw4Snww", + "UXmFTrbeIC13oIjqA5rJGdu93unPW3LQ0v944v6x2UXH5tQC+JqFcGHFHznV5KOFcCCZo7P3CEcRD6wJ", + "ZKx1pTGdpIKE3YrlDUb38WfClJgnnPp0sApzypsu8qhOJ/+6BivaGlG2JfU2dIL18A50c2tN4DmbUcFZ", + "rLWxGRZUX7OyfFZevzl+Pnz++qLV13w8TANrVDx78/Zdq9/a7fV6LR+BagpawQNfnr0/gp0yx0YlUToZ", + "SvrRIwkcZutDMYm5MBqw7YM2pmVBwZxbBJszaO2+/NkQ1/ZLoCu3KSGV0NqNYgYuU8zOy5991DKdJ0TM", + "qPSZyX7JvrmdL1zrht2XaVsSMSMiI1qg4m5B/QginoadwpTt1pgKEgisya7Vbv1BYi2Hzz5q0slh9/Tz", + "W68ayZ8rBEscJZSRJZLlNyLhXXNxFXEcdra/soDHiNJjLy7xtflQ3l9LEyQjiVZ7wRrBwmsaqukw5NdM", + "g+zhq/YLyhpnzPVGrwRH//7nvy5OczVp++UosZx2e+fJF3LaCm/VQ3tNINlC0sS/jPeJfxEXp//+57/c", + "Sh52EUYQuZVQZ/f/uRkBWLam9bD0TGmsmWW0/H1K1JSIwu3tiEX/ZPRh6I4c7RWWUjKPFt80Fxg1nxER", + "4XmB8VqYWts94H4VqARVcFZtP81Gr5DuvIIN69HcJf+yqqPv9PyM1gOUB6afNa+w90ITSDJAtndO7Z87", + "iyDVQHRFkyFIzUM8yUy2y16bz69oYkVx6GG2MYoMIwhTEN5HnKvugP19ShiCvYMNJjckAJ4nFVbo8OxE", + "omsaRWDgAaayeLVowT5nK6a5VPp/RcraaJQqLa1zRZDVm2CSFGCBxiOCUobdc3ZFdrYLrNKVRcsVEYxE", + "QyMby4aYMZ2Q7VSLHFjqGEtFhOH2aVLG1/Gvp+do43jOcEwD9KsZ9ZSHaUTQeZpofrBZxl57wBJBZlqF", + "YBMwNlI7Lx8jnqoOH3eUIMSBGMNgmYnMvrXOXp69t6/1crM7YG+JRixhIQkBZnfjSKSmWKGQsz/rE0vC", + "8rDF+StI95/ldksynMgpV0PQ/OaruNO5bX5mWq9lC2i3ZkGSlrd0p7qdr+FBXiNvRoVKcaR5bUmc9L7P", + "G88Pj9pgHEuK6ovlexl1Y1V+WG1qMDEjgxvIolDtt3sYSamx3aOgyi9YQJye+akZsCvGP2EOkKV2n1zV", + "/IK5zs0gVRTZsdtuZbfA0kmGkzKu8NdBz6EsqOa1dvWQSEWZISfdFlmJUKKNS63NWzrW+vtlG13+pfSD", + "PvtOtdDyxTUy2AB+wvRPxfGrRomV5oLmSmFlc7C8/X4cylpHJTTbRkpgJvXdqmWshHTRL8DEkSJxojkZ", + "myAqkTTMl4SI8eu/IW6EGtd1wDRo0rh5WHRkRiNJJ4yyyaYW8/XFhMPQWJbGqUqFbjejMsdmmXSc9aa6", + "gHcGOmL4cZxKfSMHURoSdOksPJdluXDR/rOoElqD0IKGY1ACmg0oe2orTpWeXi84xiqYajzxVBm/L7t0", + "WQagbGVa9R5qYcleym6x/+cZuygj1dobKoxfL86+0YBZsGCfrDMDWkHFb6K8InPYcmeOxAsGyaIl0m8v", + "FETyaEbstVu0ZY5wcGWuEuN6Yc2YxiBpbZD6+FeOqNc6t2orNL4ao7+sKiySEpiA7WJzirHSv7H/zjMu", + "pBdn5mtrxVgSQD6oHn0E4thl2+hKBCwQiGliiVBIBQnUwvCUTQYMXEAu7S9dO9qlPuRaRvEdQp+y45UF", + "C9qO6VPaWlTYWSf2wTB6aTymSpGwXZYNrghJ5OpFafHaGq491nVBrgV1jMwajMKG4hlhYy4CElsl4csU", + "x+eFwbxq3HpDLHpkGPwWYLb0hHCSRJSExv3H7AeYWaXdJ7CxVl1+w4rWZjwAylNe4ii6RBu20SYSRK9F", + "ur1inOXE/u7ozJFA9mp9cdrWFKm5wOVUqWSo/0cO9Sm+rA5m+7oTrofTd5JEBz3Qr/b2du2uWqObAbgy", + "bNm+5vVqqN8aJ37XP4zxWJ9F5ybSRJQ/yrvkltQrysKmA/yq29Za5zLByGkad22gSwTppMlEYPCQ/Zrm", + "uVs/ewI26zn4Cud3n5djhtUglYrHBV9HtFHx0KBlX44ysmY86oRYYTBlNrS3GnAX/YbjuRnK6GJ1lpjh", + "ZORx+6EfNddFEzrBo7kqvx9s93wa35e+QTtYfNtS539vNEgSDhVf7oFMx8i1beJwCPfJUPHhbEw9I2fX", + "Wu6+QiUKKs7+Vq/VQ3SSgFpzAsg4wdQ4iBokgNB4cVp8u+sOWAeu3z46zibIhs2GxCBb4tC8nGxwUQCC", + "gtcZGs03EUYXp130LoP2zxJphWVGXEDCFEs0IoShFEzPcBt2zF1cBCCVcGmqandrOzGxC5vwRMntty76", + "ZZ6QGFs7lD4KMVY0AE+nEa2sB64js1H2TRizohWskdVqmd/2WzKhUomK1zbaePviaHd391nVfrnzpNPb", + "7mw/ebfd6/f0//+juYP31w/P8I11WOYt1nesyH2O3p8c71hjaXke9XEPPzu4ucHq2T69ls8+xiMx+X0X", + "30sAh5+VHedOb2gjlUR0HJvUVOVzdSt4lNW4st3aQ+2OHM5y/9llbQ0m3umWdxGZ4vN5th6368eOVBnm", + "Sq/pwuIWNfl5AnpnfkoKEpx1Tgyo1w3zmMqrnwXBVyG/Zp57O8YTIofmPvP7M6TSONmQG2vdEJyrsTTv", + "pmWr5/be072D3f29g17PE5CxSPA8oMNA30CNAHhzdIIiPCcCQR+0AQ9eIRpFfFQm9Ce7+wdPe8+2d5rC", + "YZ54muEhU7xcL7RhMfJXF9znvpSA2tl5ur+7u9vb39/ZawSVtRc3AsrZlksiydPdp3vbBzt7jbDgE+if", + "uwCZqgAf+lwXtP5kHhs7MiEBHdMAQYgN0h3QRgxXGMleq8pncoTDoTWe+O8OhWkkl3pMmMlsS2Noi9NI", + "0SQi5htsSCNbNKz8GEbyeaNQxogYZvFDa4xkw4pWegi4tWRNUCk8rIS6UypBCsmFJ0qisG9O6Eo+B7uZ", + "A/ahjg7sGhpSwyutOnUiMiNRkQjM1aWBjbkgKKMTs2mlVVE2wxENh5QlqZckalH5IhUgi5pBER7xVJln", + "Rtiw4iTgtAy6x1iz62Z67gsurla6f+qbeChSxvQwK61Ch2BIH1tTDdziGNneLsKgIPRlz4Hm0dR+l+it", + "6WEsRPnPSaoQZYpr7ZSFo3kbZrKWJIYEkYoDJ7UGQztMU+nSL7eAsdS5f5j5ct55T74vnbFxF/i6GraY", + "EDWUCquVEoumlHfQ/hyaN/Ym1x1XGlIa4J2R6/tAOrjbdzTZdiTDyd1gfJkzWmZryBvBLSxoSLoIThd4", + "xbjwvspJO1c8SUiY2X+6A3Zujkr2kzQvKLqjwYOaEioQF3RCyxOXDWx36dW2Dik6aro1ORY7Lkqo8BHc", + "N+oPPR4rIgwGXeRyMfzIbkKr3bK4b7VblhOVUeN+9GAkd7VcAPHl2ft1fdMSwcc08iwXfCHsV6uZOa+t", + "V3u98872/2c8MDW9gYhGmfGfiHlIupXkANC+2c3z8uz9WR1MWWYGVIRuYU2Zx4uHc2R+DQ4j9lHJvkpa", + "DcaRv75Yskly2fuZT5YdCxyTUToeEzGMPca1F/o7Mg2MaxNl6PTnsjyr5eamWvNZaXNAbR7jwAbWN8O+", + "xyBXWUa7gM0P/u16S8w1XBeOp7dK2DY2Iq+LXme5MNDLs/cS5V5KHktdeXtr/eXPpnNJAxyZEU10LWVF", + "AxsQZ2MJ+SzvaE2RHjk59sqG7iCgjdkkSeEYnr/tnLy52IpDMmuXYALPoimPiIZ7s8AtZi4oL3fuLzGJ", + "WZ2lwxCGbHqACrjKTnBjJBXOqwc7iiscDWXEfc4a7/RHBB/RxsULEzSlIWijpLSV+vcCFkr0ve89MZoj", + "1U17DhNWTaalA+7VHcspZIx5pbC80qS+o/ILwZHJnFOm5zy+2208vypvNL9aeXrtIL55T5xjeIPgraPT", + "YyMwBJwpTBkRKCYK2zw9BRcXEIda7VZH31EhJjG42o3/tty7pcYEX4zGqjXiHi2k3bgTA25NuPhb44IQ", + "ohgzOiZS2XDx0sxyinee7PdNUouQjPee7He73XVjVJ7nQSmNtmLLuPAXwlW6cvpl+3AHoShN1vKpdXb4", + "7pdWv7WVSrEV8QBHW3JEWb/w7+yf+Qf4w/xzRJk3hKVRHhQ6Xsh/Un7S1HeW+b2vV8KsS5imJQ4K/Mon", + "php9BjwbIG7OGy6s8ETrJ4bivjQu+NaZQ/L0VaqQMaToENogewj9uNwS6gQjaGPnTJmiUZ5YZdEGeqvU", + "OHJp9oCFzAEJYVm+gCgyfwWczfSp8CUPKDFw9+2L3g+sl8swpB5K/rvV9oyTBERVrT5vrS2cJKvJ1i8o", + "ZvyvadIUG9rsuYkenOvf5o2tPPubyf/88f/Ls6e/b//x6uLif2cv/+f4Nf3fi+jszRdFUC2Pan/Q0PSv", + "Fo0OD0ulkPSmpHSKVeARqKZcqhoM2y9IceOv2UVHoPj1B6yDXlFFBI76aNCquAgPWmiD3OBAmV6IM6SH", + "spEOm7rzmTH/6M6fnG75uTpGaEMahN2QLJJJpqOQx5iyzQEbMDsWcguR8Kav/wpRgBOVCqJ3T8uw0RyN", + "BA7yUIZ88jb6hJPk8+aAgYZLbpTQK0iwUFkaDjcDEIWFyvgM2OYkdIHhRkMesOxeyuLCjY2mmxlBwDZf", + "9bj0I8WrvnBRDsU56Pki6MHrS29kRKUi4JidUbYmo8wdDR30SqzioHfQWyngZzS0hPzgJCwmyXRE2eAs", + "GQKGqQ3jBg+1BrZ0zZvMGUG/vHt3ptGg/3uO3EA5LrItNkqe8QGUxkaoIlnw/tts+UzfZncbLsgYyaBb", + "1CBq6LlxD3336hwpImLnsL8RaHSOaaDXB8//VMpUkyLF6PDo9Plmt0GWT8BtBv+SfXyXrbAa3GGNZnW2", + "wIziNX7b6OQY3HPtCc0FOHCrecEFigyDyc91H72XpOzrCltlXvXNTkbz3PJmboBBa9ONmFQ5RR+9zeRG", + "nIGSOVrmxOCGzM8lDGsfXozPz8LoFb9c8GayepFlbeDhg1XmJK5v3HpWsPz4ezAOZ976dRdsmuud7aIx", + "VE/mJ4187+9cWtldV0ddN0FDOYayEH+b5WhonlzhLpIULOprN1QNax/hkf5sn9ydVnJxiqZYsj8r+FjR", + "TbZ3nzbKlqlnbfp8XXy45mMDUnaqXEBm9uxqQlOvaBQZbwZJJwxH6BnaOD95+evJq1ebqIPevDmtbsWy", + "Hr79aZCrwZH2y7P3EO2C5dC9ANU7PeLccZjcUKnkYrxqo4fU5bkhfinlb/AGAG9+xaQO7vV5YRn3ka7h", + "Id36vr1UEUuTO3xphgYr7N5RgoZa5upLblDms+bnr5tq4U7AKcX++PhDUSZwPte3zm3QblGPv+mh1CyQ", + "hOjkLE9xmBul3PCVNT3b6W7vH3S3e73udq+JiS7GwZK5Tw+Pmk/e2zGGiD4e9YOwT8ZfYCK0hG2ENxxd", + "47lEAydeD1pGni8I8oVja0XwRs+viykkbpcxoipQrMoJsU4OiGbJHb40on5ZouPzcorjxkLek398UTZk", + "0vRqt74PttdwHes3QQFPo1ALUiN9dI1eRkKrPkqi8uzRcNrfsyvGr1l56cYIqhnAHykRc3RxeloymQsy", + "tslxGywcfCZq9oEna23DzgpZeyU0t8yzcB+5Fapst3DdffVMCkWbnfPBNBTawHaXi5/ed3PKzNZoOlmy", + "porVJSSzYZr6pCr9yUVevH9/clwiDoz3tw96B886B6Pt/c5e2Nvu4O3d/c7OE9wb7wZPd2vS0zf3m7m9", + "K0z5NNdHOgHiwYJpAtnCvj5vmS/LKFUo83PTB/lIi6eoIAebuB4wKpwwqiCHI2UTPQzo+FZMNgGaJs0k", + "ZVRBRgDIR0OZXjIYU/Qg1nupj15CW/iEY4g3ckBo5ahsR8Dh3NhRNWNwUyfwr+Ugn09TpeU26COnqUL6", + "X7BsjQarriwfwvCYPnrNoY9wTqaMV/Ue0xyctxabV3WkDetW5NxPYTLLMPvoRcYkMzZr2eqGJPZPw7ut", + "ZzR4fW+WfO/sjrc0teQ7V3Ara7cMRlvtlkMUuJ8tOqJZuLwxFkVS9D0wEBwBC80dfVJFI5vkAFZCpaKB", + "0RoxbG7dSbYJvUg4NCJA3XOh8R6xYkLWyTGKi1O0AeGMf0VWqdT/2syeFouncm/n2d6z/ac7z/YbBS3k", + "AK5m8Efg27QI3EpuHyTp0FX+qFn60dl7uPv0vSrT2FgJ7NoLPqKJ4IGWVilDeSmRfPJn3WfFWI2Qp6Oo", + "YHWygV0QENCk7kvN+9gfNJrR8Zj98TG42vld0Hj7Zl/ujLzKXTaRXxI+KVpKF9RGMuqYJIx+d3ogKCFr", + "I07eEgkrQOdEIaCfDsIBXNKZS5IlOReXYjHuJay93d3dg6dPdhrRlYWucHCGoL8uQnlqISgcMWiJNt6e", + "n6OtAsGZMZ2fJuSHYFaA858zZPMx90ounFp32vVRSY28lFONHXsW16L8wgpBdlEW6eBZlQlIC6fci+3d", + "3d7TvScHT5odY6uxDcXNcg7jcnIY9Ng0JsWd3wDr+rvDM6RHF2MclDWU7Z3dvSf7Tw/WgkqtBRWk4DGp", + "M9YA7ODp/pO93Z3tZqFTPgu6DQosHdgy7/IcOg9ReHbDg4pF1tuuuy18gqchsLckiDCNDwPn/VK5fUyK", + "jKEwzfJNaHIxWCPBwsXVoG8jFa1StseIBlyglGWJmbqrzaG3s27Ws2lzH6xm44sydISZRpf18TeZGG+B", + "u0SQGeWp/AoDcUUCTUzjiHOxVt86d6K3RKaRMiZIKtHF6Z+BiWjiQlKRpOwqb8lvSSTELRe31gEu0YSf", + "quuQ1Wg3mmz9sgW3a45pe5kbbOn41wYchZpVpWz10/URjoIUco/hbD/1qiB0gKcKHtrnxskjijhnKJhi", + "NiGQy91kOmQThNGUR2G35X8qicLh2PuEwa9RxE2qhCtCEpuWywChu2mZhc4I2njJ84JyhpQq6XWfxIar", + "2MRLZWp8EtcU55Q+x8EsQEnjEyteiOI3XUrafMQnErRABe4r3WrymAQL45WCmUkzN4uN8liOvNrRt70H", + "xAr39l2h5urkY6vRWhlD8QyTOBBcSkQiOoGUZhenZTCX+R/GlNFY89nVz9FlYBuQrkw4k768KHCnycbZ", + "KH0Xosex60uuRKBh8N/0Ph2Yh3zrio9izFJI1FUgZHKTUGHIo9nj+JRLNcyiSdYEVqohJGFKBclDztx9", + "OQX//blhcdDGey861nYbdFmviVv1XqAq/1B1ANbzVC9G/dhqZzToI+PFeJqlITx5TFA1AGSdiK88aw+V", + "MCotBBuhDcZViS0VMs9sNnmo8uuoep66gq6v9nrnTYOxlsdenWE1PWFj7kntuIbB33q0O9+FhIiYQhoy", + "FBJGSeiUx8zyb21b4CMfSYLClFjMGYFUYItwbI43pHBkzihG2aTC66sTNjHDGxiW52iCeW3DJk+O0u9Z", + "/U6kgCvjJCARzn2sG3k8UDn0W4oXBxZkkkZYoGrA4RKQ5TyOKLtqMrqcxyMe0QDpDtXnnDGPIn491J/k", + "T7CWzUar0x2GuYtg5XnGAGcdRM2GVObNl/CTXuVmxT0dTC9bpv8WVOxu8oLr9Rt6QSNiY/LeM3pTIPRy", + "EpO9nV5d5ELNoKWYhcV4znU5tyVZ34l3oZaHWdEDj3+a8QCqvEqUDZGl9fpWCy5my+I0Fk0xaMM9Crsk", + "MWW8FpK1NLKENPNyq7o/OGi2JAnKs+8dPHm63zBbzhfZOpfUNP4Cy+YsXmLRrNmp0yZms4MnB8+e7e49", + "ebazloHKecrU7E+dt0xxfyq1TSpGsyc9+L+1gDK+Mn6QavxlygCV6pTcGqDPS45uHiVd8+xRm6Y7Ku6k", + "e2cpW0Cb2RiXSEuHJZGrUIprg4zHBJTKocFbJwem4l3fCIYAJzigau4xmOBrk7E9a1KJ9m1iTSsD60Gp", + "HdsmbNCcS6aj3KFzw02O/mJM6xVaOGicdEumozoz/pvqrMaIn9uAik9EDV5o8roAi+aCbD3XWJa8OvTf", + "AaRczkutVf2HTIvmNaEdrWdloXPPSF/Eur8EdHH7K9tZMPuWhOQqxpddofVHcC0d2nMj+6pMrvbKrfAH", + "ewHertdwVEyHtzTfYCl3Xn7rrj9vsyJxi/3MDbb+fAUX0HU6VjODAT1aGCzK87HbJZKooSbFxeqE0HeQ", + "38f4FNwqw491R7iXJD/25ztJ7LOwHedEubbnWqNPoyX5nJkiYoY9hik3BHJNynZUw4nbyJr40Ha8Wak0", + "uDf1y2o2Areh+5iWAoeJIGN6s4RaTANzXZfdx6XFQFjO+S3RRoxv0N5TFEyxkBXYGZ1MVTQvG1n3PNET", + "X1ZCmSgtOjfPjp7vpuu4+KJht7M4uu/InhdiHfxZ20k4XBanfpQ1czbjBM9BtqxVBJ/u7vV6uzu9WwWq", + "f61k8oVx6jxCC/2sMaf09FgcIfP/XMw4eC2oqUnm0CSVIDjugzdVggOCIjKGMK4s0+tKnX5h6uXA20dS", + "6/if0b/bKFd41NpZLItjnIHg4caxMf5uGS33SluO9Sh+XwR7SbBYxmaChaixqv/qfqe32+ntv9ve7T/Z", + "729v30Vke4akOheepx+3r59GO3i8Fx3Mn/6xPX062Yl3veECd1C3oFIGsFLGwK4hIaKaSrKaglWSiDLS", + "kZnb22oH5CW8wLwkrTz/61kfzAqWCgvn5UUWZQascuRUK6rdR2CThX6pCaUK/snxcrBv5UdWBcRPYFVQ", + "gJ6aAQOZVra/NK1HyhreO+8LDRvfPEt9G1fdPT6nbzja3l2uwbiPnkuMsXTClt3Yi7eaR4WbcEHVNF5+", + "PWTNsiQB8Bj+UaqwHEjTRScTBoljiz9nbx/F2s66c6vdij7ulc+M/b15SJUNis8I0G51UQxo8DYAeYmX", + "YwGa5KqFMO4JWBBAxE/bne1n8EIffdz7qdd51kV/L3gKtA22iujbdq1Lv/aa4DBjlCB3mpfz7WdrPaM7", + "fC6joF/tvVR3EdtweUvjedZOd1c4t+nSBuefF/a4ElV0R3WCPi9ZsROc14tjd708oknXL5s8e9fbXjvr", + "znpXRPcLPIq/SNVrpt5FWKo6ufoVlirTx5BIrXTdhgo1rFKo3mYFt6EA4NoJsfZ9JCsavkuHLnOhx1kB", + "2mjCFcqDAFZKOQC+SJmXHsrw51WGIbtDLUHsrCCIZjBlgXx02dE9OUaJ4GEa5B6wEQCdBgGRcpxC0eRu", + "U4l29RvjXSrz4FquNfrVynyd9r46zJTc1O/3a3KjClNqgq3f6u3e6q2+EwtAu5Um4WoeZho142BrZeJY", + "4VPpsUeU0V6RggqL+dCAo78tYnBRwYN6SyjQ4kCauAKAmqYWKUl6yv7hm6E3R8AxiYgivkGQqchp3T6o", + "zLnoapa6vX/gN5nhm2EAAYkLgPxKSKLl9JhLWyEzxmzuBayaBBtt9FwBSIlg+I5JxGWxVQbu6UoppHar", + "mucTr1h0TcRXMX17lqHj6yYTtz1Xlnq4O0HlnVWV6jLKVH09i3bHw84/jJ0RDbv9rZ/++v92PvzlT/7S", + "KiVFShLRCckYHsCuyLxjasZqpa1bTkcK2W5aUmFbkEQRHIMVIbgixmoR45sivE962Umav8bxwhLg5TCm", + "LPv3ygX99U/1724FNL4H5rFyH+8isavijsVuxERMXPZ05+0FxbRZNNeYlqiQPc5e0+5U/llmXYplWy+N", + "aNOFYsMjCkk45YBpLQUHAUkUCbs2ixYFWASHE1UtXmyz2DnvbH3WMKTutNEylSRVn7wlgvstRq47Zoaw", + "o0ln78m+rddexOT2wg759swEXNfVN9RY9lgBXlEJ0QTOabbQGG2QOFFzl6PVuTVurhcAfpgN6H3J/MrZ", + "r3rPvkauzvdLk3N+h9U1i/H5DqCVkfkL+1+bEc/vF3VcTbRjzqStGFZODFPReKTq1LtNxfqKHoL336KL", + "k/5mPAttNspJWk3LvRUztWVz3/qiGUIoCLzUlzQ/ZS5YvQOdVrtILhUSCysrQFK/N6dOFqoWaa5H0JlG", + "zfWUCFLYCOiQJ/BcE2XWz69BjIzJUJkQ0amWlDNVDwQFx8FMjXUoyHxBF+1ayxPUnOKbbAawiWK58HIA", + "68hTtW2//Bkqmbx1pcXo2A0BYFQkVX+2mTIVNSmOvrgZRapaXLdp7z14llct4X51Z6tCnPkcJdL84LvN", + "JAlSQdVcqyNWzhkRLIg4TA0ZAn+CRcDP+eSQ4vTzZ7DLjD1uUS+10ksDdHh2AlQCtdz1ll2cooiOSTAP", + "tFYHGSoXkjqA6PHm6MRK9C6NGLxSUQUIcaVpD89OoNKlMFJtq9fd6faAxBLCcEJb/dZudxvqfmo0wBK3", + "ICM6/Gk9nk2wE+XsJLS388+mie4lcEwUEbLV/83jOayIMBnWJchCeFIQRhNMhZVGkwj8mY0eT3VfyKnj", + "GHzf3BJtg/DGBgmp5ta7iyRv7LZ+0ORgootgiTu9nhH+mbLXAc4rIG79bqPA8nkbSRmAHk+CmQVp00k6", + "FuWf26293vZa8KwsWuib9j3DqZpyQT8SAPPJmki41aQnzLicIpO8wD6qF88ZkFDxhP32Qe+XTOMYi7lD", + "V46rhMs6EY1IhKFsmknv/zsfdZE1t0JOTTnlaRSiEUHGn5aEmo1ipLDoTj4iLIIpnZEBs7eHKUCJBeQc", + "jpG+NYwwXT4aZmqz+1mk0888nFewmw23pYfrOCNbjuBq7j5JhpCCaFhXuyM3r1HGoAagJDbRYZbEfvHl", + "Hoq2yoB7q9UShpnKa4Caaq1XZG4teN4BG+UK0QwPtoVAcfAsh/XOpt9JHlIy+uNLjrNvyKK3fMkxMHsH", + "URrmkoDz28RihKPIG0w+ifgIR7ao7RXxCE4voYVFSjF7pbtyGQ+JyUSYzNWUM/N3OkqZSs3fI8GvJRH6", + "YrYZiS2ubUVHS7pQXZzGkBXY1DvQc24ZELc+XZH55+6AHYaxq2UhbUHySHJb7dfkZKESOUc/Q7v+nJk1", + "b8hHtvq/qbRZLE5owOSpSlJl388kUTaNMjSH2pVySsIBUxx9EqZU+fzz1qd8xs8gURMcajopNDFL2vpE", + "w891UMsh1qsfQlOPTkIAAYOWvl0GLf33RGAtUadyCgq2BKV6UtzSjSzAV0srm1UMB5ihhCcmOBqIyhQx", + "Lo0BKelxFCEFR8n11TIQ7GTNemy8g6++mg12MN7plWMEldYKh6m3d+A/T5IEgvjU7v85f/MawVWl98A0", + "y40ZgCPK9C2KwhTkS5i9O2DPcTBFRm6CBFiDFg0HrUzmDTcB1lRab8xOBwSvnzRoP5lp2jT8qdvVQxmZ", + "ro9++2RG6euzlMRDxa8IG7Q+t1Hhw4SqaTrKvn3wI7TOZ/y8xAjQhuH9m66gCMSu59eguTcwCxG3vDaa", + "I4xyDlTU7keUYbG0GooH9RaDWsHEE1lExqcBmOUGrf7AGeYGrfagRdgMfrPWu0Hrsx8DtnpPfbYlUxDG", + "NsuJaL/X21wdzGXx6xGhSw318fu8IH3tfDXBwwpdi4KHWZxLFad30JT2MeLWPUg+P+PQJYv/IeKtEPGs", + "Pl0Q3qB/8R4w5BsR8+JQkcC0Ah45CWypdmLIAnIlgsbhQi+NwkGdBJcTb1H9qCqZi2rFXt0pCwDEyNHf", + "3j3QH8ybl8eGeZ/d17w4gsSHWbHYx0WOsFmOENt+jfglUd8CxfXui5W6Kv4PSL+PhX5eEiv35UircLMt", + "MnOvIP4Ac3Asl3YU01jrqucAU+ecMIWew69d+1+n8UC61MuITy77yKAw4hMUUWbfiApvGPpStLiETsa3", + "POtnXc1ddp8Nc3/++5//AqAom/z7n//S0rT5C477lkm5ANlAL6cECzUiWF320a+EJB0c0Rlxi4F8fWRG", + "xBzt9kDMTAR88hQglAM2YG+JSgUrvKWZRDvSDmjr4ev1UJYSaX3zdUM6tlkAjNnTo8K7s2xQea8nur3o", + "yGhWUFiAvhUdDUBYJzUpUa3+1fJbz8yaS/azqgV3waa/mr8ocqMM9XYMgGsyGECx79zBB7totHF+/nyz", + "i0DHMFQBmR5AYs6HscJz9wdPWs2TDEcpMxTAsuFNhdrTtfbfY9ummQHYjvg9WYDrimnXm4CNyYMIEjp8", + "/dAVmpiD/XhzpmGfffbYRWTVG2hvv97iFM5FpZEi/PX22dHeIs7NlwLKHkIFRhvO+9fVgTs7OnEFRzYf", + "jOjv5dbQK7Vp+rOrA3FTfe7e1LIjzsYRDRTqOFggU3xMMlWtTCCPhR28tVAj7NZVzalWvN+2SilCam+6", + "LFtIfuXd/e1RmXSdayTP+5bT2o+bZBXpHFMZcN23QC2dACe2Cp4RX7JzWqSiVQYp40ycXTlLxSXLnk+O", + "3YG8P9OUnTpl1bvhHpjicYUhPiAjrFT2KmRKfEzU/D7bRRd9vsRy9W2RZu/+pKD7tmL5yPwxmbHCCto0", + "FzT5Ymsv0JdE/WJa3OFG2xk8Cz8nwp1ql9gWVp0ty3RFwZQEV2ZB8CC9XPc9MU2aqb5mvO9J8wX0rCOx", + "WJT/EFEaKLs5rpYpuCe2Wtnd6bcww1rq7dd757UE5kEyOJuMnMXaFALDcs6Cze/qqfdebjOD7Ed5mZ2l", + "UeRePGZEKJQVBS7eAVufwC1ptWzvTtvS6+D921cdwgIOfmiZD5VfiLJfvrKEbzbMLOUHmTTRCU0YKHX3", + "WZ2E8wX7b9wFUVZ0+r92Xtiy0/+188IUnv6v3UNTenrzzoild1+s+b4l7kdMfFrgpmWkAWtiUIBylYSa", + "tWoopLr235Wcaha9lqSa4fWHsNpEWC2ia6m8arfiTiVWM8cDPclkxObDNnxy/onfmaR6v1Y+S5EuKSyV", + "5WcPW/WDC7DzwifKUCrJI3SgpBnFFa+Nhubq/EAuvT4c6Z4ctwGRbY06yFJjA0TuyXjt4Lh34dbOe/+W", + "68N4RCcpT2Ux9iTGKpgSaYOVIlJmwI9N7M6v51rB+xum0t59Xh33Llf/oPs7kvirG2qYt3mBWiXzu1ZN", + "ZX7bXsv8tiK7iV176yq92+Q7mzVOha4cWlMyLhb69zg7+uDy6SLovVZUcnUBgQbRH7D/1vrHb4rg+MNP", + "Lkgm7fV29uF3wmYffnJxMuzUkQphSlCbEfLw9TE8+00geh2SRuYheVU4TJZ5ID2XVuU/TkHKXz6ba0iO", + "Cn9oSI00pAK6lmtIdi/uVkUqZ1a6dx3J0ZsP4Ta1xg8t6T60JJmOxzSghKm8CNOCk5it4fYIY8uYfR8q", + "OHeULtrGWlJ2KFcIoHkK8Ht37Mkmv3/lyGUbf5w+8txExYROHckvw3p95Fujh979Muf710MeM4kZgX8R", + "dYmWKX21/SD/YJwqcErMM4SA1ycSRmrPRuyivKSeTJOECyVNDkMQgKHGkppqAdiX77CcwtCXsxCyrVIi", + "2wMGWcn1ZxPLv3VF5iZDIeV5qfhspTYroS/2qpzg8UGP0deXsfzZKxvJWPd8jG2S3oeTsR6MddyLpHVS", + "yn2+kR0MUChHJDvJPAvuox8pm2w+Kg9Uw6yytRXyGXlEra2xrXLnV4FecHHVlCl4qq48At5QXOE3qH1p", + "8CB/0sMrYaCemPOjiebe+cZCKZ2HdFqnVU4SRGmoWYdjIe7yHQseD+2PJu+kPhW2ZDkodYEd9aGZjJ79", + "HlTs11whGicR0XIPCVHHUJPeTSssueTNVBYKT63HBPWxKYYQmPRd0hWvsOW54DnCbdgGvEwubpeXa0Z8", + "sjptQDa5i5H35A0YMJNcmrhM1JcoY7JIcSRJRAKFrqc0mEIOAf2bqXcH4f04SS6zpEGbffQSTmoxdxJM", + "viGJ0KJjwJnkETGpAWZxfNlfzHF5cXoKnUz6AJPN8rKPXF7L7IKQulUxJ0BWFeO1zXSwoSlJ8CgyO3qp", + "5ezC+jZttoA8qdOA+TIHMHJtB6RjdFlIInBZk0XAMdRXfCIfSpRt16fiM2tRHAlAnKFNwsJWnemaRv78", + "Ads9b8WAhrkMDBh3nMpgAZhXfJKlASyRMk6SpuRrwQQqnsXxEhpGG4XSdVKFPFV/lSokQkBnS911xI02", + "cGD+ofCVJlRbeiIr/gfk532gMXm5vKjSTLVQZ8H8axbHrXbLwlPI57WGhWFFTojqgIsPCXpnCokfftgS", + "1knpUGb2hZwOlZvDFkeuF7ltzefv3qJlERV+D3pp+QUgh4IyJ6rA3vK8Tsujig035cCrspgpKuM7I26V", + "HVkoL9fsQWChMN03oLSueifIqoxlJdDu+8FgEYLHHDYgF1YD5cvZWi8J3zwhfb0tWVhqEwr5QZvrPzk0", + "Ikwtf9dWm4NieRJK8UMFNsiEG0w5lwWyH5EpnlEubM5qW8kno0wwWRjt0fobXWpSvbTFui6teN63tiaE", + "i5/sHF3obr2U/D3cp7zHi4K2nXH8thOpIW+eRBiNBCVjlOBUEi0tpTFBpiaDTX1McDB1tVS7A/ZuSpCt", + "oFYwIGQFN6lEl9vxZRuNUoUiLCag7ZiPxvdIkIDHMWGhqYo4YFOCZ1SragJFWBEWzDuSQJXMGclLPmjV", + "3b7pIHjSyerwtZEr3wgGhstCccZLlAgCRGTUZVaqhDhgImV/M7n+9LCXDtBLRKTCo4jKaZZdP8AhYYE3", + "kd75t83Gvr4R95yoxfqFD/LKcyte+pDPPkVbZlZB9pt4EXpkri1cuDp1Ddj8EqFX1quGZV+x87xm43/g", + "kTZrdWt8oJeZDMXLTvG38SRTKtr841lG2SMZpmY6Ui5s/N2+tWQMBaWs9NxibbK3fXDJcsdnaF6L5219", + "cn+e3MJG9o1wwvaS8uH+CfJFfwss12L1Vjz3gYyD1pZUsIo9IAu2QD2c+MRFgct9E2zYHLiMGxd5jhIY", + "dCpTav0HMy69fRv3gNsyY2dxXXgAL7BnyjpJhOv4sjXO1jLgSmHv/zB/wZqy5Z8tJ3xIxpe/CNwbszvJ", + "2JtheAmeRxx/7+8yARfChMBBUN1jSsFUsAUWHpg2wOLWzjhE2/nfX5yebtZxCaGW8gihHjGHKBeCDGJP", + "fbs3MyIEDV2xvaPTY1uaj0okUtZFb2IKFfCuCEmgtAblqTTF4rvFuul1FfizwuiEKTFPOGVqJRR507sB", + "5vOtio3dM5+0Sei++8djsMI/PiYFvEOLK3YBy7VIhVWtM55zTqPMVAjU0hYe8VSPvlDXHY1pRORcKhIb", + "z7xxGsEhgjSltoqN7Wdi8NqIKon0eWhDzFJCREylpJzJARuRsRbDEiL03Lo7lEvNnYy8xnuFM655Zljf", + "t+HABqXewWcLqzqslau84yRxVd59TlJZYfpbg/QCPNKQnMcjHtEARZRdSbQR0SujdKCZRJH+Y3OpS9sQ", + "+n3tGj23P1ka0ydszL1lDAzNZsT8fYRtlNmae0R8dGztJSkeFsd/YKP9bE2u5GuC4KijaEyycGGUKhrR", + "j4bV6UGoVDQwtZ/zYDUoW2vj1QbslCih22BBUMCjiATKGVe2EsGDrUHa6+0GCYW8DrsEgAOGV/85hhmP", + "zt5DO1Natz1g+h8w8LvDM/MSO8bWRlAAlBF1zcUVOtl6s8LJ9xzQ9B/sJWcWuDRqzLvhP57v1o8FrT1D", + "suaI8mSZAsST796N00pwP6wFj9NaAMH42Wo2JgIHIBTLaapCfs38loEZj9JY/8P8cbIqpYPCwfQCmn4z", + "0q4BZ+U0boGP4lDaNYXElFl5kAcKg7DH6l+qEeeWAEJMyXPPewscqu+Rur++Ub6Ix2/wadJi1JUw+mbO", + "1n3ffBYGl6moiI/HcswNpbmVKF6xPmVpmVbqZs71dsplns0JBTjBAVXzNsJRxINcxkxlZkPqZFfuSBB8", + "pW/a7oC9zRJCWXdZrV21nWqFQiqvzAhWe+qiNzMiZDrKgEPAmIyeB8gn4YApjgIcBWmEFUFkPCYBeLpG", + "NKZK1mhfGSh3WV4on8Sz0e6jRd1jUzn8NAG7l5OFrFDcltnqLUGCCNO4qHpUkQMMEgz/YBwY6UE5Q5SN", + "I2sEDQSXEtmhOiSiEzqKrElPdtG7KUESx2TAkggzRgRKpXmd1qB3EkGkTI37tB4A6rcZimqjPBw+EVxZ", + "Y0LEuZBG/9cUfnGKpCLJEjJ7a0Y+hTXfUfo9M7id6YGulQoMZhY/yRtU6w0xlGIQrukojZR8gCdhA9BD", + "p+l7LAf/naCTCRH6VGDDZI0B3Rxrh05z6Ev+wLW5Z8+zVs1yz2ajFnz+Cv5wS8OZh67hEMTPdez0nsmv", + "aG3Eu/20no/ur7pTw7nLvqB+IOynL1zl91LS47zggtc0Y21O4Y8teWwB8tJRLbmxrg5abey3epd+pI2j", + "Ux8sKPUxx6LiknNqXdDpt0cIvfuNobjv9JWPm7ZKsaSlhPU1DvWrk7x9ExR4N9ndHjiG6BbZ3b4pr3bI", + "zvVw0UXflD+79cvO/Nl/5G+7Szd2k8QNgp3r3NgN17PPTUsVpQvbppmaZEf8niR4+0Kxhvzu0P5D62+g", + "MhSQ5TfZmQg6F91P4kTNnQmaj8HZLi+IIulHcNn1hQdnL013F5V7i0eYr0cejk5rn2B+1Km4t1eevJjf", + "yfHjL05RPHOli2VL3zodLIIpnZF6o3v5BFsUJYJ0Ep7A40poEGbx4e4yhUV38hHZ4W1GE/svRF0iTBKi", + "kAoSqGiOKFMcOIKZ488SCa41AfjOxdxnTC+e3BeCx4d2NSvuQ3umrDEs9+2N550QK9yZOW6zxIT2Be/U", + "p/jGJHahHyFY5eXPaIPcKGHyMqKx1nwQHWcoJTcBIaEEmtwsArzdq7Fs0o9kOBk1gXJJhs03NoMpClKp", + "eOz2/uQYbeBU8c6EML0XWtQfgySbCD6joSnwlSN1xiOD1e0ahK5rd9VChQ0CyZULA9yDyDBNLqTJR5qU", + "2YLxdW71WyPKMAC3Mpdl+UwZt3s9H6bg/JqfHUc5rR9XmNX8NpyyoylRKzkOiYpzk0Bp88c195ivuaL7", + "krvTSrddsxJMzTyaGjoa3UVaxczb7X7N1hffjhMOlY/S/8aazmeZQlpnNv+2SLB3f/fDfZvLLx6x0+ZL", + "4pTvgqkcBtAj+gjmFQ9whEIyIxFPoDqTadtqt1IRtfqtqVJJf2sr0u2mXKr+Qe+g1/r84fP/DQAA//+d", + "SPPbAlsBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/paths/paths.go b/lib/paths/paths.go index 524da21a..35ddf884 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -210,6 +210,16 @@ func (p *Paths) InstanceSnapshotConfig(id string) string { return filepath.Join(p.InstanceSnapshotLatest(id), "config.json") } +// InstanceSnapshotSchedule returns the path to the snapshot schedule config. +func (p *Paths) InstanceSnapshotSchedule(id string) string { + return filepath.Join(p.SnapshotSchedulesDir(), id+".json") +} + +// SnapshotSchedulesDir returns the root directory for instance snapshot schedules. +func (p *Paths) SnapshotSchedulesDir() string { + return filepath.Join(p.dataDir, "snapshot-schedules") +} + // GuestsDir returns the root guests directory. func (p *Paths) GuestsDir() string { return filepath.Join(p.dataDir, "guests") diff --git a/lib/scheduledsnapshots/README.md b/lib/scheduledsnapshots/README.md new file mode 100644 index 00000000..99670f74 --- /dev/null +++ b/lib/scheduledsnapshots/README.md @@ -0,0 +1,24 @@ +# Scheduled Snapshots + +This feature lets users attach a periodic snapshot policy to an instance. + +- A schedule runs on a fixed interval (for example `1h` or `24h`). +- New schedules establish cadence at `now + interval + deterministic jitter` derived from the instance ID. +- Updating only retention, metadata, or `name_prefix` preserves `next_run_at`; changing `interval` establishes a new cadence. +- Each run auto-selects snapshot behavior from current instance state: + - `Running` or `Standby` source -> `Standby` snapshot + - `Stopped` source -> `Stopped` snapshot +- For running sources, scheduled capture includes a brief pause/resume cycle during snapshot creation. +- The minimum interval is `1m`, but running/standby captures briefly pause/resume the guest, so larger intervals are recommended for heavier workloads. +- Each schedule stores runtime status (`next_run_at`, `last_run_at`, `last_snapshot_id`, `last_error`). + +Retention cleanup is required per schedule and can use either or both: + +- `max_count`: keep only the newest N scheduled snapshots for that instance. +- `max_age`: delete scheduled snapshots older than a duration. + +Cleanup only applies to snapshots created by that schedule for that same source instance. +Manual snapshots are never deleted by scheduled retention. + +If the source instance is deleted, the schedule is kept so retention can continue cleaning previously created scheduled snapshots. +Once no scheduled snapshots remain for that deleted instance, the scheduler removes the stale schedule automatically. diff --git a/lib/scheduledsnapshots/policy.go b/lib/scheduledsnapshots/policy.go new file mode 100644 index 00000000..1eea696d --- /dev/null +++ b/lib/scheduledsnapshots/policy.go @@ -0,0 +1,103 @@ +package scheduledsnapshots + +import ( + "fmt" + "hash/fnv" + "time" + + "github.com/kernel/hypeman/lib/tags" +) + +func ValidateSetRequest(req SetRequest, validateName func(name string) error) error { + if req.Interval < MinInterval { + return fmt.Errorf("interval must be at least %s", MinInterval) + } + if req.NamePrefix != "" { + if validateName == nil { + return fmt.Errorf("name validator is required") + } + if err := validateName(req.NamePrefix); err != nil { + return err + } + if len(req.NamePrefix) > MaxNamePrefixLength { + return fmt.Errorf("name_prefix must be at most %d characters", MaxNamePrefixLength) + } + } + if err := tags.Validate(req.Metadata); err != nil { + return err + } + if req.Retention.MaxCount < 0 { + return fmt.Errorf("retention.max_count must be >= 0") + } + if req.Retention.MaxAge < 0 { + return fmt.Errorf("retention.max_age must be >= 0") + } + if req.Retention.MaxCount == 0 && req.Retention.MaxAge == 0 { + return fmt.Errorf("retention.max_count or retention.max_age must be set") + } + return nil +} + +func BuildSnapshotMetadata(instanceID string, userMetadata tags.Tags) tags.Tags { + metadata := tags.Clone(userMetadata) + if metadata == nil { + metadata = make(tags.Tags) + } + metadata[MetadataKeyScheduled] = "true" + metadata[MetadataKeySourceInstanceID] = instanceID + return metadata +} + +func IsScheduledSnapshot(metadata tags.Tags, instanceID string) bool { + if metadata == nil { + return false + } + if metadata[MetadataKeyScheduled] != "true" { + return false + } + return metadata[MetadataKeySourceInstanceID] == instanceID +} + +func BuildSnapshotName(prefix string, runAt time.Time) string { + if prefix == "" { + prefix = DefaultNamePrefix + } + return prefix + "-" + runAt.UTC().Format(NameTimestampFormat) +} + +func InitialNextRunAt(instanceID string, interval time.Duration, now time.Time) time.Time { + return now.UTC().Add(interval).Add(cadenceJitter(instanceID, interval)) +} + +func NextRun(previous time.Time, interval time.Duration, now time.Time) time.Time { + if interval <= 0 { + return now + } + if previous.IsZero() { + return now.Add(interval) + } + if now.Before(previous) { + return previous + } + + steps := int64(now.Sub(previous)/interval) + 1 + return previous.Add(time.Duration(steps) * interval) +} + +func cadenceJitter(instanceID string, interval time.Duration) time.Duration { + if interval <= 0 { + return 0 + } + + maxJitter := interval / 10 + if maxJitter > maxCadenceJitter { + maxJitter = maxCadenceJitter + } + if maxJitter <= 0 { + return 0 + } + + hasher := fnv.New64a() + _, _ = hasher.Write([]byte(instanceID)) + return time.Duration(hasher.Sum64() % uint64(maxJitter)) +} diff --git a/lib/scheduledsnapshots/policy_test.go b/lib/scheduledsnapshots/policy_test.go new file mode 100644 index 00000000..65e0ddb1 --- /dev/null +++ b/lib/scheduledsnapshots/policy_test.go @@ -0,0 +1,116 @@ +package scheduledsnapshots + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateSetRequestNamePrefixLimit(t *testing.T) { + validateName := func(name string) error { + if name == "" { + return assert.AnError + } + return nil + } + + req := SetRequest{ + Interval: time.Hour, + NamePrefix: strings.Repeat("a", MaxNamePrefixLength+1), + Retention: Retention{ + MaxCount: 1, + }, + } + err := ValidateSetRequest(req, validateName) + require.Error(t, err) + assert.Contains(t, err.Error(), "name_prefix must be at most") +} + +func TestMarshalUnmarshalScheduleRoundTrip(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + last := now.Add(-time.Hour) + id := "snap-1" + in := &Schedule{ + InstanceID: "inst-1", + Interval: 2 * time.Hour, + NamePrefix: "nightly", + Metadata: map[string]string{"env": "test"}, + Retention: Retention{MaxCount: 3, MaxAge: 24 * time.Hour}, + NextRunAt: now, + LastRunAt: &last, + LastSnapshotID: &id, + CreatedAt: now.Add(-2 * time.Hour), + UpdatedAt: now, + } + + raw, err := MarshalSchedule(in) + require.NoError(t, err) + + out, err := UnmarshalSchedule(raw) + require.NoError(t, err) + assert.Equal(t, in.InstanceID, out.InstanceID) + assert.Equal(t, in.Interval, out.Interval) + assert.Equal(t, in.NamePrefix, out.NamePrefix) + assert.Equal(t, in.Retention.MaxCount, out.Retention.MaxCount) + assert.Equal(t, in.Retention.MaxAge, out.Retention.MaxAge) + require.NotNil(t, out.LastRunAt) + assert.Equal(t, *in.LastRunAt, *out.LastRunAt) + require.NotNil(t, out.LastSnapshotID) + assert.Equal(t, *in.LastSnapshotID, *out.LastSnapshotID) +} + +func TestMarshalSchedulePersistsZeroMaxCount(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + in := &Schedule{ + InstanceID: "inst-1", + Interval: time.Hour, + Retention: Retention{ + MaxCount: 0, + MaxAge: 24 * time.Hour, + }, + NextRunAt: now, + CreatedAt: now, + UpdatedAt: now, + } + + raw, err := MarshalSchedule(in) + require.NoError(t, err) + assert.Contains(t, string(raw), "\"max_count\": 0") + + out, err := UnmarshalSchedule(raw) + require.NoError(t, err) + assert.Equal(t, 0, out.Retention.MaxCount) + assert.Equal(t, 24*time.Hour, out.Retention.MaxAge) +} + +func TestNextRunUsesIntervalStepCount(t *testing.T) { + previous := time.Date(2026, 3, 10, 10, 0, 0, 0, time.UTC) + interval := time.Hour + + // 2.5 intervals later should advance to the next whole interval boundary. + now := previous.Add(2*time.Hour + 30*time.Minute) + next := NextRun(previous, interval, now) + assert.Equal(t, previous.Add(3*time.Hour), next) +} + +func TestInitialNextRunAtUsesDeterministicBoundedJitter(t *testing.T) { + now := time.Date(2026, 3, 10, 10, 0, 0, 0, time.UTC) + + first := InitialNextRunAt("inst-1", 24*time.Hour, now) + second := InitialNextRunAt("inst-1", 24*time.Hour, now.Add(3*time.Hour)) + + firstJitter := first.Sub(now.Add(24 * time.Hour)) + secondJitter := second.Sub(now.Add(27 * time.Hour)) + + assert.Equal(t, firstJitter, secondJitter) + assert.GreaterOrEqual(t, firstJitter, time.Duration(0)) + assert.Less(t, firstJitter, 5*time.Minute) + + shortInterval := 15 * time.Minute + shortJitter := InitialNextRunAt("inst-1", shortInterval, now).Sub(now.Add(shortInterval)) + assert.GreaterOrEqual(t, shortJitter, time.Duration(0)) + assert.Less(t, shortJitter, shortInterval/10) +} diff --git a/lib/scheduledsnapshots/storage.go b/lib/scheduledsnapshots/storage.go new file mode 100644 index 00000000..11f3e271 --- /dev/null +++ b/lib/scheduledsnapshots/storage.go @@ -0,0 +1,118 @@ +package scheduledsnapshots + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "strings" + "time" + + "github.com/kernel/hypeman/lib/tags" +) + +type storageModel struct { + InstanceID string `json:"instance_id"` + Interval string `json:"interval"` + NamePrefix string `json:"name_prefix,omitempty"` + Metadata tags.Tags `json:"metadata,omitempty"` + Retention retentionStorage `json:"retention"` + NextRunAt time.Time `json:"next_run_at"` + LastRunAt *time.Time `json:"last_run_at,omitempty"` + LastSnapshotID *string `json:"last_snapshot_id,omitempty"` + LastError *string `json:"last_error,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type retentionStorage struct { + MaxCount int `json:"max_count"` + MaxAge string `json:"max_age,omitempty"` +} + +func MarshalSchedule(schedule *Schedule) ([]byte, error) { + if schedule == nil { + return nil, fmt.Errorf("nil snapshot schedule") + } + maxAge := "" + if schedule.Retention.MaxAge > 0 { + maxAge = schedule.Retention.MaxAge.String() + } + + model := storageModel{ + InstanceID: schedule.InstanceID, + Interval: schedule.Interval.String(), + NamePrefix: schedule.NamePrefix, + Metadata: tags.Clone(schedule.Metadata), + Retention: retentionStorage{MaxCount: schedule.Retention.MaxCount, MaxAge: maxAge}, + NextRunAt: schedule.NextRunAt.UTC(), + LastRunAt: schedule.LastRunAt, + LastSnapshotID: schedule.LastSnapshotID, + LastError: schedule.LastError, + CreatedAt: schedule.CreatedAt.UTC(), + UpdatedAt: schedule.UpdatedAt.UTC(), + } + return json.MarshalIndent(model, "", " ") +} + +func UnmarshalSchedule(content []byte) (*Schedule, error) { + var model storageModel + if err := json.Unmarshal(content, &model); err != nil { + return nil, fmt.Errorf("unmarshal snapshot schedule: %w", err) + } + + interval, err := time.ParseDuration(model.Interval) + if err != nil { + return nil, fmt.Errorf("parse schedule interval %q: %w", model.Interval, err) + } + + var maxAge time.Duration + if model.Retention.MaxAge != "" { + maxAge, err = time.ParseDuration(model.Retention.MaxAge) + if err != nil { + return nil, fmt.Errorf("parse schedule retention max_age %q: %w", model.Retention.MaxAge, err) + } + } + + return &Schedule{ + InstanceID: model.InstanceID, + Interval: interval, + NamePrefix: model.NamePrefix, + Metadata: tags.Clone(model.Metadata), + Retention: Retention{MaxCount: model.Retention.MaxCount, MaxAge: maxAge}, + NextRunAt: model.NextRunAt, + LastRunAt: model.LastRunAt, + LastSnapshotID: model.LastSnapshotID, + LastError: model.LastError, + CreatedAt: model.CreatedAt, + UpdatedAt: model.UpdatedAt, + }, nil +} + +func ListInstanceIDs(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read snapshot schedules directory: %w", err) + } + + instanceIDs := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if !strings.HasSuffix(name, ".json") { + continue + } + instanceID := strings.TrimSuffix(name, ".json") + if instanceID == "" { + continue + } + instanceIDs = append(instanceIDs, instanceID) + } + sort.Strings(instanceIDs) + return instanceIDs, nil +} diff --git a/lib/scheduledsnapshots/types.go b/lib/scheduledsnapshots/types.go new file mode 100644 index 00000000..371615d6 --- /dev/null +++ b/lib/scheduledsnapshots/types.go @@ -0,0 +1,47 @@ +package scheduledsnapshots + +import ( + "time" + + "github.com/kernel/hypeman/lib/tags" +) + +const ( + MetadataKeyScheduled = "hypeman.scheduled" + MetadataKeySourceInstanceID = "hypeman.schedule_instance_id" + DefaultNamePrefix = "scheduled" + NameTimestampFormat = "20060102-150405" + MaxSnapshotNameLength = 63 + MaxNamePrefixLength = MaxSnapshotNameLength - len(NameTimestampFormat) - 1 + MinInterval = time.Minute + maxCadenceJitter = 5 * time.Minute +) + +// Retention defines automatic cleanup rules for scheduled snapshots. +type Retention struct { + MaxCount int // Keep at most this many scheduled snapshots for the instance (0 = unlimited) + MaxAge time.Duration // Delete scheduled snapshots older than this age (0 = unlimited) +} + +// Schedule defines periodic snapshot capture for a single instance. +type Schedule struct { + InstanceID string + Interval time.Duration + NamePrefix string + Metadata tags.Tags + Retention Retention + NextRunAt time.Time + LastRunAt *time.Time + LastSnapshotID *string + LastError *string + CreatedAt time.Time + UpdatedAt time.Time +} + +// SetRequest configures or updates a schedule for an instance. +type SetRequest struct { + Interval time.Duration + NamePrefix string + Metadata tags.Tags + Retention Retention +} diff --git a/lib/scopes/scopes.go b/lib/scopes/scopes.go index be2a7fed..ec621926 100644 --- a/lib/scopes/scopes.go +++ b/lib/scopes/scopes.go @@ -234,6 +234,9 @@ var RouteScopes = map[string]Scope{ "POST /instances/{id}/fork": InstanceWrite, "GET /instances/{id}/logs": InstanceRead, "POST /instances/{id}/restore": InstanceWrite, + "DELETE /instances/{id}/snapshot-schedule": SnapshotDelete, + "GET /instances/{id}/snapshot-schedule": SnapshotRead, + "PUT /instances/{id}/snapshot-schedule": SnapshotWrite, "POST /instances/{id}/snapshots": SnapshotWrite, "POST /instances/{id}/snapshots/{snapshotId}/restore": SnapshotWrite, "POST /instances/{id}/standby": InstanceWrite, diff --git a/lib/snapshot/README.md b/lib/snapshot/README.md index b8233c72..82cd4300 100644 --- a/lib/snapshot/README.md +++ b/lib/snapshot/README.md @@ -121,6 +121,37 @@ If the native binary is missing or not executable, Hypeman falls back to the in- - Removes snapshot metadata and payload. - Does not modify source or forked instances. +## Scheduled Snapshots + +Per-instance schedules can create snapshots automatically on an interval. + +- Configure with: + - `PUT /instances/{id}/snapshot-schedule` +- Inspect with: + - `GET /instances/{id}/snapshot-schedule` +- Disable with: + - `DELETE /instances/{id}/snapshot-schedule` + +### Schedule Rules +- Schedules do not take a snapshot kind input. +- Each run auto-selects behavior from current source state: + - `Running`/`Standby` source -> `Standby` snapshot + - `Stopped` source -> `Stopped` snapshot +- `Standby` scheduled runs against a `Running` source include a brief pause/resume cycle during capture. +- `interval` uses Go duration format (for example `1h`, `24h`). +- The minimum interval is `1m`, but larger intervals are recommended for heavier or latency-sensitive workloads because running captures pause/resume the guest. +- `retention` is required and must set at least one of: + - `max_count`: keep only the newest N scheduled snapshots (`0` disables count-based cleanup) + - `max_age`: delete scheduled snapshots older than a duration +- Optional `name_prefix` (max 47 chars) and `metadata` are applied to each scheduled snapshot. +- New schedules establish cadence at `now + interval + deterministic jitter` derived from the instance ID. +- Updating only retention, metadata, or `name_prefix` preserves `next_run_at`; changing `interval` establishes a new cadence. + +### Cleanup Scope +- Retention cleanup only targets snapshots created by the schedule for that same instance. +- Manually created snapshots are never deleted by schedule retention. +- If the source instance is deleted, the schedule remains until scheduled snapshots for that instance are gone, then self-deletes. + ## Safety Rules - Snapshot creation rejects writable volume attachments. diff --git a/openapi.yaml b/openapi.yaml index 0c41698d..bf93406f 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -553,6 +553,94 @@ components: Optional hypervisor override. Allowed only when forking from a Stopped snapshot. Standby snapshots must fork with their original hypervisor. example: cloud-hypervisor + + SnapshotScheduleRetention: + type: object + description: Automatic cleanup policy for scheduled snapshots. + properties: + max_count: + type: integer + minimum: 0 + description: Keep at most this many scheduled snapshots for the instance (0 disables count-based cleanup). + example: 7 + max_age: + type: string + description: Delete scheduled snapshots older than this duration (Go duration format). + example: 168h + + SetSnapshotScheduleRequest: + type: object + required: [interval, retention] + properties: + interval: + type: string + description: Snapshot interval (Go duration format, minimum 1m). + example: 24h + name_prefix: + type: string + description: Optional prefix for auto-generated scheduled snapshot names (max 47 chars). + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + maxLength: 47 + nullable: true + example: nightly + metadata: + $ref: "#/components/schemas/Tags" + retention: + $ref: "#/components/schemas/SnapshotScheduleRetention" + description: At least one of max_count or max_age must be provided. + + SnapshotSchedule: + type: object + required: [instance_id, interval, retention, next_run_at, created_at, updated_at] + properties: + instance_id: + type: string + description: Source instance ID. + example: qilviffnqzck2jrim1x6s2b1 + interval: + type: string + description: Snapshot interval (Go duration format). + example: 24h + name_prefix: + type: string + nullable: true + description: Optional prefix used for generated scheduled snapshot names. + example: nightly + metadata: + $ref: "#/components/schemas/Tags" + retention: + $ref: "#/components/schemas/SnapshotScheduleRetention" + next_run_at: + type: string + format: date-time + description: Next scheduled run time. + example: "2026-03-10T02:00:00Z" + last_run_at: + type: string + format: date-time + nullable: true + description: Last schedule execution time. + example: "2026-03-09T02:00:00Z" + last_snapshot_id: + type: string + nullable: true + description: Snapshot ID produced by the last successful run. + example: q7z1w7l2af4l8y7q1h7g2m3s + last_error: + type: string + nullable: true + description: Last schedule run error, if any. + example: "invalid state transition: stopped snapshot requires source in Stopped, got Running" + created_at: + type: string + format: date-time + description: Schedule creation timestamp. + example: "2026-03-09T01:00:00Z" + updated_at: + type: string + format: date-time + description: Schedule update timestamp. + example: "2026-03-09T01:30:00Z" Instance: type: object @@ -2202,6 +2290,118 @@ paths: schema: $ref: "#/components/schemas/Error" + /instances/{id}/snapshot-schedule: + get: + summary: Get snapshot schedule for an instance + operationId: getInstanceSnapshotSchedule + security: + - bearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: string + description: Source instance ID or name + responses: + 200: + description: Snapshot schedule + content: + application/json: + schema: + $ref: "#/components/schemas/SnapshotSchedule" + 404: + description: Snapshot schedule not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + put: + summary: Create or update snapshot schedule for an instance + description: | + Scheduled runs automatically choose snapshot behavior from current instance state: + - `Running` or `Standby` source: create a `Standby` snapshot. + - `Stopped` source: create a `Stopped` snapshot. + For running instances, this includes a brief pause/resume cycle during each capture. + The minimum supported interval is `1m`, but larger intervals are recommended for + heavier or latency-sensitive workloads. + Updating only retention, metadata, or `name_prefix` preserves the next scheduled + run; changing `interval` establishes a new cadence. + operationId: setInstanceSnapshotSchedule + security: + - bearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: string + description: Source instance ID or name + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/SetSnapshotScheduleRequest" + responses: + 200: + description: Snapshot schedule updated + content: + application/json: + schema: + $ref: "#/components/schemas/SnapshotSchedule" + 400: + description: Bad request - invalid schedule request + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 404: + description: Instance not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + delete: + summary: Delete snapshot schedule for an instance + operationId: deleteInstanceSnapshotSchedule + security: + - bearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: string + description: Source instance ID or name + responses: + 204: + description: Snapshot schedule deleted + 404: + description: Snapshot schedule not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /instances/{id}/snapshots/{snapshotId}/restore: post: summary: Restore an instance from a snapshot in-place diff --git a/stainless.yaml b/stainless.yaml index dbd1ed2c..82c264cb 100644 --- a/stainless.yaml +++ b/stainless.yaml @@ -72,6 +72,9 @@ resources: instances: models: snapshot_policy: "#/components/schemas/SnapshotPolicy" + snapshot_schedule: "#/components/schemas/SnapshotSchedule" + snapshot_schedule_retention: "#/components/schemas/SnapshotScheduleRetention" + set_snapshot_schedule_request: "#/components/schemas/SetSnapshotScheduleRequest" volume_mount: "#/components/schemas/VolumeMount" port_mapping: "#/components/schemas/PortMapping" instance: "#/components/schemas/Instance" @@ -102,6 +105,11 @@ resources: methods: create: post /instances/{id}/snapshots restore: post /instances/{id}/snapshots/{snapshotId}/restore + snapshot_schedule: + methods: + get: get /instances/{id}/snapshot-schedule + update: put /instances/{id}/snapshot-schedule + delete: delete /instances/{id}/snapshot-schedule snapshots: models: