feat(logs): archive command history daily
Some checks failed
Verify / verify (push) Failing after 17s
Some checks failed
Verify / verify (push) Failing after 17s
This commit is contained in:
@@ -5,6 +5,7 @@ Maintainarr is a self-hosted Go application for managing a single organization's
|
||||
It ships with:
|
||||
|
||||
- SQLite-backed persistence
|
||||
- Daily compressed command log archives
|
||||
- User registration and login
|
||||
- OTP-based 2FA setup and verification
|
||||
- Role model with `admin`, `editor`, and `viewer`
|
||||
@@ -66,6 +67,7 @@ No example users, groups, nodes, or jobs are seeded.
|
||||
```env
|
||||
MAINTAINARR_ADDR=:8080
|
||||
MAINTAINARR_DB_PATH=data/maintainarr.db
|
||||
MAINTAINARR_LOG_ARCHIVE_DIR=data/log-archives
|
||||
MAINTAINARR_SESSION_KEY=change-me-session-key-please
|
||||
MAINTAINARR_ENCRYPTION_KEY=change-me-encryption-key-32bytes
|
||||
MAINTAINARR_ORG_NAME=Maintainarr
|
||||
|
||||
@@ -128,7 +128,7 @@ func New() (*App, error) {
|
||||
})
|
||||
})
|
||||
|
||||
scheduler := services.NewSchedulerService(database, nodes)
|
||||
scheduler := services.NewSchedulerService(database, nodes, cfg.LogArchiveDir)
|
||||
if err := scheduler.Start(context.Background(), org.ID, cfg.RefreshCron); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
type Config struct {
|
||||
Address string
|
||||
DatabasePath string
|
||||
LogArchiveDir string
|
||||
SessionKey string
|
||||
EncryptionKey string
|
||||
OrgName string
|
||||
@@ -21,6 +22,7 @@ func Load() Config {
|
||||
return Config{
|
||||
Address: env("MAINTAINARR_ADDR", ":8080"),
|
||||
DatabasePath: env("MAINTAINARR_DB_PATH", filepath.Join("data", "maintainarr.db")),
|
||||
LogArchiveDir: env("MAINTAINARR_LOG_ARCHIVE_DIR", filepath.Join("data", "log-archives")),
|
||||
SessionKey: env("MAINTAINARR_SESSION_KEY", "change-me-session-key-please"),
|
||||
EncryptionKey: env("MAINTAINARR_ENCRYPTION_KEY", "change-me-encryption-key-32bytes"),
|
||||
OrgName: env("MAINTAINARR_ORG_NAME", "Maintainarr"),
|
||||
|
||||
@@ -188,6 +188,16 @@ func migrate(ctx context.Context, database *sql.DB) error {
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (monitor_id) REFERENCES uptime_monitors(id)
|
||||
);`,
|
||||
`CREATE TABLE IF NOT EXISTS log_archives (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
organization_id INTEGER NOT NULL,
|
||||
archive_date TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
entry_count INTEGER NOT NULL DEFAULT 0,
|
||||
compressed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE (organization_id, archive_date),
|
||||
FOREIGN KEY (organization_id) REFERENCES organizations(id)
|
||||
);`,
|
||||
}
|
||||
|
||||
for _, statement := range statements {
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -1199,13 +1202,15 @@ type SchedulerService struct {
|
||||
cron *cron.Cron
|
||||
nodeService *NodeService
|
||||
db *sql.DB
|
||||
archiveDir string
|
||||
}
|
||||
|
||||
func NewSchedulerService(database *sql.DB, nodeService *NodeService) *SchedulerService {
|
||||
func NewSchedulerService(database *sql.DB, nodeService *NodeService, archiveDir string) *SchedulerService {
|
||||
return &SchedulerService{
|
||||
cron: cron.New(),
|
||||
nodeService: nodeService,
|
||||
db: database,
|
||||
archiveDir: archiveDir,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1239,6 +1244,7 @@ func (s *SchedulerService) Start(ctx context.Context, orgID int64, refreshSpec s
|
||||
if _, err := s.cron.AddFunc("@daily", func() {
|
||||
_ = s.nodeService.RefreshAllNodeInventory(context.Background(), orgID)
|
||||
_ = s.nodeService.ScanAllNodeUpdates(context.Background(), orgID)
|
||||
_ = s.nodeService.ArchiveCommandRuns(context.Background(), orgID, time.Now().AddDate(0, 0, -1), s.archiveDir)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1387,6 +1393,187 @@ func (s *NodeService) logCommandRun(ctx context.Context, params commandRunParams
|
||||
`, params.JobID, params.NodeID, params.Action, params.CommandText, params.Status, params.Output, params.TriggeredBy, startedAt, finishedAt)
|
||||
}
|
||||
|
||||
type archivedCommandRun struct {
|
||||
ID int64
|
||||
GroupName string
|
||||
NodeName string
|
||||
Action string
|
||||
CommandText string
|
||||
Status string
|
||||
Output string
|
||||
StartedAt time.Time
|
||||
FinishedAt *time.Time
|
||||
UserName string
|
||||
}
|
||||
|
||||
func (s *NodeService) ArchiveCommandRuns(ctx context.Context, orgID int64, day time.Time, archiveDir string) error {
|
||||
if strings.TrimSpace(archiveDir) == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
dayStart := time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, time.Local)
|
||||
dayEnd := dayStart.AddDate(0, 0, 1)
|
||||
archiveDate := dayStart.Format("2006-01-02")
|
||||
|
||||
var alreadyArchived int
|
||||
if err := s.db.QueryRowContext(ctx, `
|
||||
SELECT COUNT(1)
|
||||
FROM log_archives
|
||||
WHERE organization_id = ? AND archive_date = ?
|
||||
`, orgID, archiveDate).Scan(&alreadyArchived); err != nil {
|
||||
return err
|
||||
}
|
||||
if alreadyArchived > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT cr.id, COALESCE(g.name, ''), COALESCE(n.name, ''), cr.action, cr.command_text, cr.status, cr.output,
|
||||
cr.started_at, cr.finished_at, COALESCE(u.name, '')
|
||||
FROM command_runs cr
|
||||
LEFT JOIN nodes n ON n.id = cr.node_id
|
||||
LEFT JOIN vm_groups g ON g.id = n.group_id
|
||||
LEFT JOIN automation_jobs j ON j.id = cr.job_id
|
||||
LEFT JOIN users u ON u.id = cr.triggered_by
|
||||
WHERE (n.organization_id = ? OR j.organization_id = ?)
|
||||
AND cr.started_at >= ? AND cr.started_at < ?
|
||||
ORDER BY cr.started_at ASC, cr.id ASC
|
||||
`, orgID, orgID, dayStart, dayEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var runs []archivedCommandRun
|
||||
for rows.Next() {
|
||||
var run archivedCommandRun
|
||||
if err := rows.Scan(
|
||||
&run.ID, &run.GroupName, &run.NodeName, &run.Action, &run.CommandText, &run.Status, &run.Output,
|
||||
&run.StartedAt, &run.FinishedAt, &run.UserName,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
runs = append(runs, run)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(runs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(archiveDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
archivePath := filepath.Join(archiveDir, fmt.Sprintf("command-runs-%s.log.gz", archiveDate))
|
||||
if _, err := os.Stat(archivePath); err == nil {
|
||||
return fmt.Errorf("log archive already exists for %s", archiveDate)
|
||||
} else if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
tempPath := archivePath + ".tmp"
|
||||
file, err := os.Create(tempPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
success := false
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
if !success {
|
||||
_ = os.Remove(tempPath)
|
||||
}
|
||||
}()
|
||||
|
||||
gzipWriter := gzip.NewWriter(file)
|
||||
for _, run := range runs {
|
||||
target := run.NodeName
|
||||
if strings.TrimSpace(run.GroupName) != "" {
|
||||
target = run.GroupName + " / " + target
|
||||
}
|
||||
commandText := strings.TrimSpace(run.CommandText)
|
||||
if commandText == "" {
|
||||
commandText = strings.TrimSpace(run.Action)
|
||||
}
|
||||
duration := formatDuration(run.StartedAt, run.FinishedAt)
|
||||
finishedAt := ""
|
||||
if run.FinishedAt != nil {
|
||||
finishedAt = run.FinishedAt.Format(time.RFC3339)
|
||||
}
|
||||
output := strings.TrimSpace(run.Output)
|
||||
if output == "" {
|
||||
output = "-"
|
||||
}
|
||||
|
||||
if _, err := fmt.Fprintf(gzipWriter,
|
||||
"[%s] %d\nTarget: %s\nAction: %s\nCommand: %s\nStatus: %s\nDuration: %s\nFinished: %s\nTriggered By: %s\nOutput:\n%s\n\n",
|
||||
run.StartedAt.Format(time.RFC3339),
|
||||
run.ID,
|
||||
serviceDefaultIfEmpty(target, "Unknown"),
|
||||
serviceDefaultIfEmpty(strings.TrimSpace(run.Action), "-"),
|
||||
serviceDefaultIfEmpty(commandText, "-"),
|
||||
serviceDefaultIfEmpty(strings.TrimSpace(run.Status), "-"),
|
||||
serviceDefaultIfEmpty(duration, "-"),
|
||||
serviceDefaultIfEmpty(finishedAt, "-"),
|
||||
serviceDefaultIfEmpty(strings.TrimSpace(run.UserName), "system"),
|
||||
output,
|
||||
); err != nil {
|
||||
_ = gzipWriter.Close()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := gzipWriter.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tempPath, archivePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
_ = os.Remove(archivePath)
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO log_archives (organization_id, archive_date, file_path, entry_count, compressed_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`, orgID, archiveDate, archivePath, len(runs), time.Now()); err != nil {
|
||||
_ = os.Remove(archivePath)
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
DELETE FROM command_runs
|
||||
WHERE id IN (
|
||||
SELECT cr.id
|
||||
FROM command_runs cr
|
||||
LEFT JOIN nodes n ON n.id = cr.node_id
|
||||
LEFT JOIN automation_jobs j ON j.id = cr.job_id
|
||||
WHERE (n.organization_id = ? OR j.organization_id = ?)
|
||||
AND cr.started_at >= ? AND cr.started_at < ?
|
||||
)
|
||||
`, orgID, orgID, dayStart, dayEnd); err != nil {
|
||||
_ = os.Remove(archivePath)
|
||||
return err
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
_ = os.Remove(archivePath)
|
||||
return err
|
||||
}
|
||||
|
||||
success = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func sanitizeCommand(command string) string {
|
||||
trimmed := strings.TrimSpace(command)
|
||||
if trimmed == "" {
|
||||
@@ -1407,6 +1594,13 @@ func sanitizeCommand(command string) string {
|
||||
return sanitized
|
||||
}
|
||||
|
||||
func serviceDefaultIfEmpty(value, fallback string) string {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func updateScanCommand(packageManager string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(packageManager)) {
|
||||
case "apt":
|
||||
|
||||
Reference in New Issue
Block a user