From 27396c2c7c65a81554bf828cab0bae793f540c43 Mon Sep 17 00:00:00 2001 From: Yann Stepienik Date: Sat, 8 Feb 2025 15:45:09 +0000 Subject: [PATCH] [release] v0.18.0-unstable7 --- .gitignore | 2 +- changelog.md | 2 + client/src/api/backup.ts | 82 ++- client/src/api/cron.jsx | 10 + client/src/api/storage.jsx | 10 + client/src/components/filePicker.jsx | 28 +- client/src/components/newFileModal.jsx | 77 +++ .../src/components/tabbedView/tabbedView.jsx | 17 +- .../MainLayout/Header/HeaderContent/jobs.jsx | 32 +- .../Header/HeaderContent/restartMenu.jsx | 7 +- client/src/menu-items/pages.jsx | 1 + client/src/pages/backups/backupDialog.jsx | 209 +++++++ client/src/pages/backups/backups.jsx | 147 +++++ client/src/pages/backups/fileExplorer.jsx | 249 ++++++++ client/src/pages/backups/index.jsx | 36 ++ client/src/pages/backups/overview.jsx | 186 ++++++ client/src/pages/backups/repositories.jsx | 89 +++ client/src/pages/backups/restore.jsx | 101 ++++ client/src/pages/backups/restoreDialog.jsx | 127 +++++ .../src/pages/backups/single-backup-index.jsx | 54 ++ .../src/pages/backups/single-repo-index.jsx | 124 ++++ client/src/pages/config/users/configman.jsx | 42 +- client/src/pages/config/users/restart.jsx | 120 ++-- client/src/pages/cron/jobsManage.jsx | 1 + client/src/routes/MainRoutes.jsx | 9 +- client/src/utils/locales/en/translation.json | 15 +- package.json | 2 +- rclone.sh | 1 + src/CRON.go | 3 + src/backups/API.go | 530 ++++++++++++++++++ src/backups/index.go | 92 +++ src/backups/restic.go | 461 +++++++++++++++ src/configapi/set.go | 2 + src/cron/API.go | 19 + src/cron/index.go | 71 ++- src/cron/tracker.go | 42 ++ src/httpServer.go | 7 + src/index.go | 2 +- src/storage/filepicker.go | 76 +++ src/storage/rclone.go | 5 + src/system.go | 7 +- src/utils/types.go | 3 + src/utils/utils.go | 4 + 43 files changed, 3033 insertions(+), 71 deletions(-) create mode 100644 client/src/components/newFileModal.jsx create mode 100644 client/src/pages/backups/backupDialog.jsx create mode 100644 client/src/pages/backups/backups.jsx create mode 100644 client/src/pages/backups/fileExplorer.jsx create mode 100644 client/src/pages/backups/index.jsx create mode 100644 client/src/pages/backups/overview.jsx create mode 100644 client/src/pages/backups/repositories.jsx create mode 100644 client/src/pages/backups/restore.jsx create mode 100644 client/src/pages/backups/restoreDialog.jsx create mode 100644 client/src/pages/backups/single-backup-index.jsx create mode 100644 client/src/pages/backups/single-repo-index.jsx create mode 100644 rclone.sh create mode 100644 src/backups/API.go create mode 100644 src/backups/index.go create mode 100644 src/backups/restic.go create mode 100644 src/cron/tracker.go diff --git a/.gitignore b/.gitignore index 8b17a66..d89f33d 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,4 @@ restic-arm rclone rclone-arm test-backup -backups \ No newline at end of file +/backups \ No newline at end of file diff --git a/changelog.md b/changelog.md index 8c77a28..2c79930 100644 --- a/changelog.md +++ b/changelog.md @@ -3,6 +3,8 @@ - Implements sudo mode - your normal token last longer, but you need to "sudo" to do admin tasks - Re-Implements the SSO using openID internally - fixes issue where you need to re-loging when app are on different domains (because of browser cookies limitations) - Implements local HTTPS Certificate Authority, to locally trust self-signed certificates on devices + - Added new folder button to file picker + - Fixed bug with RClone storage duplication in the UI - Implements hybrid HTTPS with public and self-signed certificates switched on the fly - OpenID now returns more info in case of errors when Cosmos is in debug mode - Localizations improvements (Thanks @madejackson) diff --git a/client/src/api/backup.ts b/client/src/api/backup.ts index bf86a3d..e008705 100644 --- a/client/src/api/backup.ts +++ b/client/src/api/backup.ts @@ -25,6 +25,24 @@ function listSnapshots(name: string) { })) } +function listSnapshotsFromRepo(name: string) { + return wrap(fetch(`/cosmos/api/backups-repository/${name}/snapshots`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json' + }, + })) +} + +function listRepo() { + return wrap(fetch(`/cosmos/api/backups-repository`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json' + }, + })) +} + function listFolders(name: string, snapshot: string, path?: string) { return wrap(fetch(`/cosmos/api/backups/${name}/${snapshot}/folders?path=${path || '/'}`, { method: 'GET', @@ -54,6 +72,16 @@ function addBackup(config: BackupConfig) { })) } +function editBackup(config: BackupConfig) { + return wrap(fetch('/cosmos/api/backups/edit', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(config) + })) +} + function removeBackup(name: string, deleteRepo: boolean = false) { return wrap(fetch(`/cosmos/api/backups/${name}`, { method: 'DELETE', @@ -64,6 +92,51 @@ function removeBackup(name: string, deleteRepo: boolean = false) { })) } +function forgetSnapshot(name: string, snapshot: string, deleteRepo: boolean = false) { + return wrap(fetch(`/cosmos/api/backups/${name}/${snapshot}/forget`, { + method: 'DELETE', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ deleteRepo }) + })) +} + +function subfolderRestoreSize(name: string, snapshot: string, path: string) { + return wrap(fetch(`/cosmos/api/backups/${name}/${snapshot}/subfolder-restore-size?path=` +encodeURIComponent(path), { + method: 'GET', + headers: { + 'Content-Type': 'application/json' + }, + })) +} + +function backupNow(name: string) { + return wrap(fetch('/cosmos/api/jobs/run', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + scheduler: "Restic", + name: "Restic backup " + name, + }) + })) +} + +function forgetNow(name: string) { + return wrap(fetch('/cosmos/api/jobs/run', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + scheduler: "Restic", + name: "Restic forget " + name, + }) + })) +} + export { listSnapshots, listFolders, @@ -71,5 +144,12 @@ export { addBackup, removeBackup, BackupConfig, - RestoreConfig + RestoreConfig, + listRepo, + listSnapshotsFromRepo, + forgetSnapshot, + editBackup, + backupNow, + forgetNow, + subfolderRestoreSize }; \ No newline at end of file diff --git a/client/src/api/cron.jsx b/client/src/api/cron.jsx index 3aeb7d8..791d705 100644 --- a/client/src/api/cron.jsx +++ b/client/src/api/cron.jsx @@ -68,6 +68,15 @@ function deleteJob(name) { })) } +function runningJobs() { + return wrap(fetch('/cosmos/api/jobs/running', { + method: 'GET', + headers: { + 'Content-Type': 'application/json' + } + })) +} + export { listen, list, @@ -75,4 +84,5 @@ export { stop, get, deleteJob, + runningJobs } diff --git a/client/src/api/storage.jsx b/client/src/api/storage.jsx index b410715..bda28c6 100644 --- a/client/src/api/storage.jsx +++ b/client/src/api/storage.jsx @@ -255,10 +255,20 @@ const listDir = (storage, path) => { })) } +const newFolder = (storage, path, folder) => { + return wrap(fetch(`/cosmos/api/new-dir?storage=${storage}&path=${path}&folder=${folder}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + })) +} + export { mounts, disks, snapRAID, raid, + newFolder, listDir, }; \ No newline at end of file diff --git a/client/src/components/filePicker.jsx b/client/src/components/filePicker.jsx index b6464e4..3bcc0d6 100644 --- a/client/src/components/filePicker.jsx +++ b/client/src/components/filePicker.jsx @@ -15,6 +15,7 @@ import { FolderOpenOutlined, ClockCircleOutlined, DashboardOutlined, DeleteOutli import * as API from '../api'; import { current } from '@reduxjs/toolkit'; import { json } from 'react-router'; +import { NewFolderButton } from './newFileModal'; function transformToTree(data) { const root = { @@ -57,7 +58,7 @@ function transformToTree(data) { return root; } -const FilePickerModal = ({ raw, open, cb, OnClose, onPick, _storage = '', _path = '', select='any' }) => { +const FilePickerModal = ({ raw, open, cb, OnClose, onPick, canCreate, _storage = '', _path = '', select='any' }) => { const { t } = useTranslation(); const [selectedStorage, setSelectedStorage] = React.useState(_storage); const [selectedPath, setSelectedPath] = React.useState(_path); @@ -68,11 +69,11 @@ const FilePickerModal = ({ raw, open, cb, OnClose, onPick, _storage = '', _path let explore = !onPick; - const updateFiles = (storage, path) => { + const updateFiles = (storage, path, force) => { let resetStorage = storage != selectedStorage; // if already loaded, just update the tree - if(files[path]) { + if(!force && files[path]) { setSelectedStorage(storage); setSelectedPath(path); return; @@ -194,13 +195,22 @@ const FilePickerModal = ({ raw, open, cb, OnClose, onPick, _storage = '', _path ))} - + }
- {!explore &&
{t('mgmt.storage.selected')}: {selectedFullPath}
} + {!explore && +
{canCreate ? { + updateFiles(selectedStorage, selectedPath, true); + setTimeout(() => { + setSelectedFullPath(res); + }, 1000); + } + } storage={selectedStorage} path={selectedFullPath}/> : null}
+
{t('mgmt.storage.selected')}: {selectedFullPath}
+ } {convertToTreeView(directoriesAsTree)}
@@ -231,15 +241,15 @@ const FilePickerModal = ({ raw, open, cb, OnClose, onPick, _storage = '', _path ); }; -const FilePickerButton = ({ raw, onPick, size = '100%', storage = '', path = '', select="any" }) => { +const FilePickerButton = ({ disabled, raw, canCreate, onPick, size = '100%', storage = '', path = '', select="any" }) => { const [open, setOpen] = React.useState(false); return ( <> - setOpen(true)}> + setOpen(true)} disabled={disabled}> {onPick ? <> : } - {open && setOpen(false)} _storage={storage} _path={path} />} + {open && setOpen(false)} _storage={storage} _path={path} />} ); } diff --git a/client/src/components/newFileModal.jsx b/client/src/components/newFileModal.jsx new file mode 100644 index 0000000..fe5e056 --- /dev/null +++ b/client/src/components/newFileModal.jsx @@ -0,0 +1,77 @@ +// material-ui +import * as React from 'react'; +import { Alert, Box, Button, Checkbox, CircularProgress, FormControl, FormHelperText, Grid, Icon, IconButton, InputLabel, List, ListItem, ListItemButton, MenuItem, NativeSelect, Select, Stack, TextField, Typography } from '@mui/material'; +import Dialog from '@mui/material/Dialog'; +import {TreeItem, TreeView} from '@mui/x-tree-view'; +import DialogActions from '@mui/material/DialogActions'; +import DialogContent from '@mui/material/DialogContent'; +import DialogContentText from '@mui/material/DialogContentText'; +import DialogTitle from '@mui/material/DialogTitle'; +import { LoadingButton } from '@mui/lab'; +import { useFormik, FormikProvider } from 'formik'; +import * as Yup from 'yup'; +import { useTranslation } from 'react-i18next'; +import { FolderOpenOutlined, ClockCircleOutlined, DashboardOutlined, DeleteOutlined, DownOutlined, LockOutlined, SafetyOutlined, UpOutlined, ExpandOutlined, FolderFilled, FileFilled, FileOutlined, LoadingOutlined, FolderAddFilled, FolderViewOutlined } from "@ant-design/icons"; +import * as API from '../api'; +import { current } from '@reduxjs/toolkit'; +import { json } from 'react-router'; + +const NewFolderModal = ({ raw, open, cb, OnClose, onPick, canCreate, storage = '', path = '/', select = 'any' }) => { + const { t } = useTranslation(); + + const formik = useFormik({ + initialValues: { + name: '', + }, + onSubmit: (values) => { + API.storage.newFolder(storage, path, values.name).then((res) => { + OnClose(); + cb && cb(res.data.created); + }); + }, + }); + + return ( + +
+ {t('mgmt.storage.newFolderIn')} {storage}:{path} + + + + + + + + + +
+ + ); +}; + +const NewFolderButton = ({ cb, storage = '', path = '/' }) => { + const [open, setOpen] = React.useState(false); + const { t } = useTranslation(); + + return ( + <> + + {open && setOpen(false)} storage={storage} path={path} />} + + ); +} + +export default NewFolderModal; +export { NewFolderButton }; \ No newline at end of file diff --git a/client/src/components/tabbedView/tabbedView.jsx b/client/src/components/tabbedView/tabbedView.jsx index a0bdc88..355fb4f 100644 --- a/client/src/components/tabbedView/tabbedView.jsx +++ b/client/src/components/tabbedView/tabbedView.jsx @@ -49,7 +49,7 @@ const PrettyTabbedView = ({ rootURL, tabs, isLoading, currentTab, setCurrentTab useEffect(() => { if (!rootURL) return; - if (initialMount) { + // if (initialMount) { // On initial mount, check if URL matches any tab const currentPath = decodeURIComponent(window.location.pathname).replace(/\/$/, ''); const matchingTabIndex = tabs.findIndex(tab => { @@ -61,18 +61,27 @@ const PrettyTabbedView = ({ rootURL, tabs, isLoading, currentTab, setCurrentTab } setInitialMount(false); - } else { + // } + // else { // After initial mount, update URL when value changes - window.history.pushState({}, '', `${rootURL}${tabs[value].url}`); - } + // window.history.pushState({}, '', `${rootURL}${tabs[value].url}`); + // } }, [rootURL, tabs, value]); const handleChange = (event, newValue) => { + if (rootURL) { + window.history.pushState({}, '', `${rootURL}${tabs[newValue].url}`); + } + setValue(newValue); setCurrentTab && setCurrentTab(newValue); }; const handleSelectChange = (event) => { + if (rootURL) { + window.history.pushState({}, '', `${rootURL}${tabs[event.target.value].url}`); + } + setValue(event.target.value); setCurrentTab && setCurrentTab(event.target.value); }; diff --git a/client/src/layout/MainLayout/Header/HeaderContent/jobs.jsx b/client/src/layout/MainLayout/Header/HeaderContent/jobs.jsx index fb0028e..4bfe819 100644 --- a/client/src/layout/MainLayout/Header/HeaderContent/jobs.jsx +++ b/client/src/layout/MainLayout/Header/HeaderContent/jobs.jsx @@ -27,7 +27,7 @@ import dayjs from 'dayjs'; import MainCard from '../../../../components/MainCard'; import Transitions from '../../../../components/@extended/Transitions'; // assets -import { BellOutlined, ClockCircleOutlined, CloseOutlined, CloseSquareOutlined, ExclamationCircleOutlined, GiftOutlined, InfoCircleOutlined, LoadingOutlined, MessageOutlined, PlaySquareOutlined, SettingOutlined, WarningOutlined } from '@ant-design/icons'; +import { BellOutlined, ClockCircleOutlined, CloseOutlined, CloseSquareOutlined, ExclamationCircleOutlined, GiftOutlined, InfoCircleOutlined, LoadingOutlined, MessageOutlined, PlaySquareOutlined, ReloadOutlined, SettingOutlined, WarningOutlined } from '@ant-design/icons'; import * as API from '../../../../api'; import { redirectToLocal } from '../../../../utils/indexs'; @@ -66,6 +66,8 @@ const Jobs = () => { const matchesXs = useMediaQuery(theme.breakpoints.down('md')); const [jobs, setJobs] = useState([]); const [from, setFrom] = useState(''); + const anchorRef = useRef(null); + const [open, setOpen] = useState(false); const statusColor = (job) => { return { @@ -76,8 +78,9 @@ const Jobs = () => { }[getStatus(job)] } - const refreshJobs = () => { + const refreshJobs = (force) => { if (!isAdmin) return; + if (!force && !open) return; API.cron.list().then((res) => { setJobs(() => res.data); @@ -89,18 +92,20 @@ const Jobs = () => { const interval = setInterval(() => { refreshJobs(); - }, 20000); + }, 5000); return () => clearInterval(interval); }, []); - const anchorRef = useRef(null); - const [open, setOpen] = useState(false); const handleToggle = () => { - refreshJobs(); + refreshJobs(true); setOpen((prevOpen) => !prevOpen); }; + + const handleRefresh = () => { + refreshJobs(true); + }; const handleClose = (event) => { if (anchorRef.current && anchorRef.current.contains(event.target)) { @@ -117,10 +122,16 @@ const Jobs = () => { let flatList = []; jobs && Object.values(jobs).forEach((list) => list && Object.values(list).forEach(job => flatList.push(job))); flatList.sort((a, b) => { + const getTimestamp = (date) => new Date(date).getTime() || 0; + if (a.Running && !b.Running) return -1; if (!a.Running && b.Running) return 1; - if (a.Running && b.Running) return a.LastStarted - b.LastStarted; - return a.LastRun - b.LastRun; + + if (a.Running && b.Running) { + return getTimestamp(a.LastStarted) - getTimestamp(b.LastStarted); + } + + return getTimestamp(b.LastRun) - getTimestamp(a.LastRun); }); if (!isAdmin) return null; @@ -179,9 +190,12 @@ const Jobs = () => { border={false} content={false} secondary={ + <> + + - + } > { const { t } = useTranslation(); const [openResartModal, setOpenRestartModal] = React.useState(false); const [openRestartServerModal, setOpenRestartServerModal] = React.useState(false); const [status, setStatus] = React.useState({}); + const {role} = useClientInfos(); + const isAdmin = role === "2"; React.useEffect(() => { API.getStatus().then((res) => { @@ -21,7 +24,7 @@ const RestartMenu = () => { const restartServer = API.restartServer; - return <> + return isAdmin ? <> @@ -33,7 +36,7 @@ const RestartMenu = () => { {t('global.restartCosmos')} - + : <>; }; export default RestartMenu; \ No newline at end of file diff --git a/client/src/menu-items/pages.jsx b/client/src/menu-items/pages.jsx index cd2a0bc..82c99d0 100644 --- a/client/src/menu-items/pages.jsx +++ b/client/src/menu-items/pages.jsx @@ -32,6 +32,7 @@ const pages = { type: 'item', url: '/cosmos-ui/backups', icon: CloudServerOutlined, + adminOnly: true }, { id: 'url', diff --git a/client/src/pages/backups/backupDialog.jsx b/client/src/pages/backups/backupDialog.jsx new file mode 100644 index 0000000..cbf7e37 --- /dev/null +++ b/client/src/pages/backups/backupDialog.jsx @@ -0,0 +1,209 @@ +import { LoadingButton } from "@mui/lab"; +import { Alert, Grid, Button, Dialog, DialogActions, DialogContent, DialogContentText, DialogTitle, FormHelperText, Stack, TextField } from "@mui/material"; +import React, { useState } from "react"; +import { FormikProvider, useFormik } from "formik"; +import * as yup from "yup"; +import * as API from '../../api'; +import ResponsiveButton from "../../components/responseiveButton"; +import { PlusCircleOutlined } from "@ant-design/icons"; +import { crontabToText } from "../../utils/indexs"; +import { Trans, useTranslation } from 'react-i18next'; +import { FilePickerButton } from '../../components/filePicker'; +import { CosmosInputText } from "../config/users/formShortcuts"; + +const isAbsolutePath = (path) => path.startsWith('/') || /^[a-zA-Z]:\\/.test(path); // Unix & Windows support + +const BackupDialogInternal = ({ refresh, open, setOpen, data = {} }) => { + const { t } = useTranslation(); + const isEdit = Boolean(data.Name); + const formik = useFormik({ + initialValues: { + name: data.Name || t('mgmt.backup.newBackup'), + source: data.Source || '', + repository: data.Repository || '/backups', + crontab: data.Crontab || '0 0 4 * * *', + crontabForget: data.CrontabForget || '0 0 12 * * *', + retentionPolicy: data.RetentionPolicy || '--keep-last 3 --keep-daily 7 --keep-weekly 8 --keep-yearly 3' + }, + validationSchema: yup.object({ + name: yup + .string() + .required(t('global.required')) + .min(3, t('mgmt.backup.min3chars')), + + source: yup + .string() + .required(t('global.required')) + .test('is-absolute', t('global.absolutePath'), isAbsolutePath), + + repository: yup + .string() + .required(t('global.required')) + .test('is-absolute', t('global.absolutePath'), isAbsolutePath), + + crontab: yup.string().required(t('global.required')), + crontabForget: yup.string().required(t('global.required')), + }), + onSubmit: (values, { setErrors, setStatus, setSubmitting }) => { + console.log(values); + setSubmitting(true); + (data.Name ? API.backups.editBackup({ + ...values + }) : API.backups.addBackup({ + ...values + })).then(() => { + setStatus({ success: true }); + setSubmitting(false); + setOpen(false); + refresh && refresh(); + }).catch((err) => { + setStatus({ success: false }); + setErrors({ submit: err.message }); + setSubmitting(false); + }); + }, + }); + + return ( + setOpen(false)}> + +
+ + {data.Name ? (t('global.edit') + " " + data.Name) : t('mgmt.backup.createBackup')} + + + + + + + + + + + {!isEdit && { + if(path) + formik.setFieldValue('source', path); + }} + size="150%" + select="folder" + />} + + + + + {!isEdit && { + if(path) + formik.setFieldValue('repository', path); + }} + canCreate={true} + size="150%" + select="folder" + />} + + + + + + + + + + {formik.errors.submit && ( + + {formik.errors.submit} + + )} + + + + + + + {data.Name ? t('global.update') : t('global.createAction')} + + +
+ + + ); +}; + +const BackupDialog = ({ refresh, data }) => { + const { t } = useTranslation(); + const [open, setOpen] = useState(false); + + return ( + <> + {open && } +
+ {!data ? ( + setOpen(true)} + variant="contained" + size="small" + startIcon={} + > + {t('mgmt.backup.newBackup')} + + ) : ( +
setOpen(true)}>{t('global.edit')}
+ )} +
+ + ); +}; + +export default BackupDialog; +export { BackupDialog, BackupDialogInternal }; \ No newline at end of file diff --git a/client/src/pages/backups/backups.jsx b/client/src/pages/backups/backups.jsx new file mode 100644 index 0000000..c5a165c --- /dev/null +++ b/client/src/pages/backups/backups.jsx @@ -0,0 +1,147 @@ +import React from "react"; +import { useEffect, useState } from "react"; +import * as API from "../../api"; +import PrettyTableView from "../../components/tableView/prettyTableView"; +import { CloudOutlined, CloudServerOutlined, DeleteOutlined, EditOutlined, ReloadOutlined } from "@ant-design/icons"; +import { Checkbox, CircularProgress, ListItemIcon, ListItemText, MenuItem, Stack } from "@mui/material"; +import { crontabToText } from "../../utils/indexs"; +import MenuButton from "../../components/MenuButton"; +import ResponsiveButton from "../../components/responseiveButton"; +import { useTranslation } from "react-i18next"; +import { ConfirmModalDirect } from "../../components/confirmModal"; +import BackupDialog, { BackupDialogInternal } from "./backupDialog"; + +export const Backups = () => { + const { t } = useTranslation(); + // const [isAdmin, setIsAdmin] = useState(false); + let isAdmin = true; + const [config, setConfig] = useState(null); + const [backups, setBackups] = useState([]); + const [loading, setLoading] = useState(false); + const [deleteBackup, setDeleteBackup] = useState(null); + const [editOpened, setEditOpened] = useState(null); + + const refresh = async () => { + setLoading(true); + let configAsync = await API.config.get(); + setConfig(configAsync.data); + setBackups(configAsync.data.Backup.Backups || []); + setLoading(false); + }; + + const apiDeleteBackup = async (name) => { + setLoading(true); + await API.backups.removeBackup(name, true); + setLoading(false); + setDeleteBackup(null); + refresh(); + } + + const tryDeleteBackup = async (name) => { + setDeleteBackup(name); + } + + useEffect(() => { + refresh(); + }, []); + + return <> + {(config) ? <> + {deleteBackup && apiDeleteBackup(deleteBackup)} + onClose={() => setDeleteBackup(null)} + />} + + + + } onClick={refresh}> + {t('global.refresh')} + + +
+ {editOpened && } + {backups && '/cosmos-ui/backups/' + r.Name} + data={Object.values(backups)} + getKey={(r) => r.Name} + columns={[ + { + title: '', + field: () => , + style: { + textAlign: 'right', + width: '64px', + }, + }, + { + title: t('global.name'), + field: (r) => r.Name, + underline: true, + }, + { + title: t('mgmt.backup.sourceTitle'), + field: (r) => r.Source, + underline: true, + }, + { + title: t('mgmt.backup.repositoryTitle'), + field: (r) => r.Repository, + underline: true, + }, + { + title: t('mgmt.backup.scheduleTitle'), + screenMin: 'sm', + field: (r) => crontabToText(r.Crontab, t), + underline: true, + }, + { + title: '', + clickable:true, + field: (r) => { + return
+ + { + window.open('/cosmos-ui/backups/' + r.Name, '_blank'); + }}> + + + + {t('global.open')} + + setEditOpened(r)}> + + + + {t('global.edit')} + + tryDeleteBackup(r.name)}> + + + + {t('global.delete')} + + +
+ } + } + ]} + />} + {!backups && +
+ +
+ } +
+ + :
+ +
} + ; +}; \ No newline at end of file diff --git a/client/src/pages/backups/fileExplorer.jsx b/client/src/pages/backups/fileExplorer.jsx new file mode 100644 index 0000000..336c640 --- /dev/null +++ b/client/src/pages/backups/fileExplorer.jsx @@ -0,0 +1,249 @@ +// material-ui +import * as React from 'react'; +import { Alert, Box, Button, Checkbox, CircularProgress, FormControl, FormHelperText, Grid, Icon, IconButton, InputLabel, List, ListItem, ListItemButton, MenuItem, NativeSelect, Select, Stack, TextField, Typography } from '@mui/material'; +import Dialog from '@mui/material/Dialog'; +import {TreeItem, TreeView} from '@mui/x-tree-view'; +import DialogActions from '@mui/material/DialogActions'; +import DialogContent from '@mui/material/DialogContent'; +import DialogContentText from '@mui/material/DialogContentText'; +import DialogTitle from '@mui/material/DialogTitle'; +import { LoadingButton } from '@mui/lab'; +import { useFormik, FormikProvider } from 'formik'; +import * as Yup from 'yup'; +import { useTranslation } from 'react-i18next'; +import { FolderOpenOutlined, ClockCircleOutlined, DashboardOutlined, DeleteOutlined, DownOutlined, LockOutlined, SafetyOutlined, UpOutlined, ExpandOutlined, FolderFilled, FileFilled, FileOutlined, LoadingOutlined, FolderAddFilled, FolderViewOutlined, ReloadOutlined } from "@ant-design/icons"; +import * as API from '../../api'; +import {simplifyNumber} from '../dashboard/components/utils'; +import { current } from '@reduxjs/toolkit'; +import { json } from 'react-router'; +import ResponsiveButton from '../../components/responseiveButton'; +import RestoreDialog from './restoreDialog'; + +function pathIncluded(list, path, originalSource) { + if(list.includes(path) || list.includes(originalSource)) { + return true; + } + + const pathParts = path.split('/'); + return list.some(p => { + const parentParts = p.split('/'); + return parentParts.every((part, i) => pathParts[i] === part); + }); +} + +function metaData(file) { + // get latest change between mtime and ctime and return with dayjs time ago + const mtime = file.mtime; + const ctime = file.ctime; + + let latest; + + let res = ""; + + if (file.size) { + res += simplifyNumber(file.size, 'B'); + + if(mtime && ctime) { + res += " | "; + } + } + + if(mtime && ctime) { + if (mtime > ctime) { + latest = mtime; + } else { + latest = ctime; + } + + res += new Date(latest).toLocaleString(); + } + + return res; +} + + +const BackupFileExplorer = ({ getFile, onSelect, selectedSnapshot, backupName, backup, className = '' }) => { + const [files, setFiles] = React.useState({}); + const [directoriesAsTree, setDirectoriesAsTree] = React.useState({}); + const [selectedPath, setSelectedPath] = React.useState(''); + const [candidatePaths, setCandidatePaths] = React.useState([]); + const { t } = useTranslation(); + + function transformToTree(data) { + const root = { + path: Object.keys(data)[0], + label: Object.keys(data)[0].split('/').pop(), + opened: true, + file: { + type: "dir", + }, + children: [] + }; + + root.label = backup.Source + + function buildTree(node) { + const path = node.path; + if (data[path]) { + for (const item of data[path]) { + const childPath = `${path}/${item.name}`.replace(/^\/\//, '/'); + const child = { + path: childPath, + label: item.name, + file: item, + }; + + if (data[childPath]) { + child.children = []; + buildTree(child); + } + + node.children.push(child); + } + } + } + + buildTree(root); + return root; + } + + React.useEffect(() => { + updateFiles(backup.Source, true); + }, [selectedSnapshot]); + + const updateFiles = async (path, forceNew) => { + // If already loaded, just update the selection + if (!forceNew && files[path]) { + setSelectedPath(path); + return; + } + + if(forceNew) { + setCandidatePaths([]); + } + + try { + const result = await getFile(path); + if (!result.data) return; + + const metadata = result.data.shift(); + const files = result.data.filter(i => i.path != path); + + setFiles(current => { + let newFiles = { + ...current, + [path]: files || [] + }; + if(forceNew) { + newFiles = { + [path]: files || [] + }; + } + setDirectoriesAsTree(transformToTree(newFiles)); + return newFiles; + }); + setSelectedPath(path); + } catch (error) { + console.error('Error loading files:', error); + } + }; + + const handleToggleNode = (event, nodePath) => { + nodePath = nodePath.split('@').pop(); + console.log(nodePath) + if (nodePath === backup.Source) { + updateFiles(nodePath); + return; + } + + // Find the parent path and filename + let nodeParent = nodePath.split('/').slice(0, -1).join('/'); + nodeParent = nodeParent.replace(/\/$/, ''); + if (nodeParent === '') { + nodeParent = '/'; + } + const filename = nodePath.split('/').pop(); + + if (files[nodeParent]) { + const file = files[nodeParent].find(f => f.name === filename); + if (file) { + if (file.type == "dir") { + updateFiles(nodePath); + } + onSelect && onSelect(nodePath); + setSelectedPath(nodePath); + } + } + + return true; + }; + + const renderAllTree = (node) => { + let nodeCounter = 0; + + const renderTree = (node) => { + nodeCounter++; + return ( + + + {nodeCounter > 1 ? { + e.stopPropagation(); + if(candidatePaths.some(path => node.path == path)) { + setCandidatePaths(candidatePaths.filter(path => node.path != path)); + } else { + setCandidatePaths([...candidatePaths, node.path]); + } + }} + checked={pathIncluded(candidatePaths, node.path)} + disabled={pathIncluded(candidatePaths, node.path) && !candidatePaths.some(path => node.path == path)} + /> : } + {node.label} + +
{metaData(node.file)}
+ + } + icon={node.file && node.file.type == "dir" ? null : } + > + {Array.isArray(node.children) + ? node.children.map((child) => renderTree(child)) + : ( + (node.file && node.file.type == "dir") ? + } /> + : null + )} + + ); + }; + + return renderTree(node); + } + + if(Object.keys(directoriesAsTree).length === 0) { + return ( + + + + ); + } + + return ( + + + + + } + defaultExpandIcon={} + > + {Object.keys(directoriesAsTree).length > 0 && renderAllTree(directoriesAsTree)} + + + ); +}; + +export default BackupFileExplorer; \ No newline at end of file diff --git a/client/src/pages/backups/index.jsx b/client/src/pages/backups/index.jsx new file mode 100644 index 0000000..8286bd9 --- /dev/null +++ b/client/src/pages/backups/index.jsx @@ -0,0 +1,36 @@ +import React, { useEffect, useState } from 'react'; +import { useParams } from 'react-router-dom'; +import { Stack, CircularProgress, Card, Typography, useMediaQuery, Chip, Button } from '@mui/material'; +import { CloudServerOutlined, CalendarOutlined, FolderOutlined, ReloadOutlined, InfoCircleOutlined, CloudOutlined } from '@ant-design/icons'; +import * as API from '../../api'; +import { crontabToText } from '../../utils/indexs'; +import ResponsiveButton from '../../components/responseiveButton'; +import MainCard from '../../components/MainCard'; +import { useTranslation } from 'react-i18next'; +import BackupOverview from './overview'; +import PrettyTabbedView from '../../components/tabbedView/tabbedView'; +import Back from '../../components/back'; +import BackupRestore from './restore'; +import EventExplorerStandalone from '../dashboard/eventsExplorerStandalone'; +import { Backups } from './backups'; +import { Repositories } from './repositories'; + +export default function AllBackupsIndex() { + const { t } = useTranslation(); + + return
+ + + }, + { + title: t('mgmt.backup.repositories'), + children: + }, + ]} /> + +
; +} \ No newline at end of file diff --git a/client/src/pages/backups/overview.jsx b/client/src/pages/backups/overview.jsx new file mode 100644 index 0000000..247cbc1 --- /dev/null +++ b/client/src/pages/backups/overview.jsx @@ -0,0 +1,186 @@ +import React, { useEffect, useState } from 'react'; +import { Link, useParams } from 'react-router-dom'; +import { Stack, CircularProgress, Card, Typography, useMediaQuery, Chip, Button } from '@mui/material'; +import { CloudServerOutlined, CalendarOutlined, FolderOutlined, ReloadOutlined, InfoCircleOutlined, CloudOutlined, UpCircleOutlined, DeleteOutlined } from '@ant-design/icons'; +import * as API from '../../api'; +import { crontabToText } from '../../utils/indexs'; +import ResponsiveButton from '../../components/responseiveButton'; +import MainCard from '../../components/MainCard'; +import { useTranslation } from 'react-i18next'; + +export default function BackupOverview({backupName}) { + const [backup, setBackup] = useState(null); + const [loading, setLoading] = useState(true); + const [snapshots, setSnapshots] = useState([]); + const [isNotFound, setIsNotFound] = useState(false); + const isMobile = useMediaQuery((theme) => theme.breakpoints.down('sm')); + const { t } = useTranslation(); + const [taskQueued, setTaskQueued] = useState(0); + + const infoStyle = { + backgroundColor: 'rgba(0, 0, 0, 0.1)', + padding: '10px', + borderRadius: '5px', + } + + const getChip = (backup, snapshots) => { + if (snapshots.length == 0) { + return ; + } + else if ((new Date(snapshots[0].time).getTime()) < Date.now() - 1000 * 60 * 60 * 24 * 60) { + return ; + } + else { + return ; + } + } + + const fetchBackupDetails = async () => { + setLoading(true); + try { + const configResponse = await API.config.get(); + const backupData = configResponse.data.Backup.Backups[backupName]; + + if (!backupData) { + setIsNotFound(true); + return; + } + + setBackup(backupData); + + // Fetch snapshots for this backup + const snapshotsResponse = await API.backups.listSnapshots(backupName); + if(snapshotsResponse.data && snapshotsResponse.data.reverse) { + snapshotsResponse.data.reverse(); + } + setSnapshots(snapshotsResponse.data || []); + } catch (error) { + console.error('Error fetching backup details:', error); + } + setLoading(false); + }; + + useEffect(() => { + fetchBackupDetails(); + }, [backupName]); + + if (loading) { + return ( + + + + ); + } + + if (!backup) { + return ( +
+ + Backup not found: {backupName} + +
+ ); + } + + function backupNow() { + API.backups.backupNow(backupName).then(() => { + setTaskQueued((new Date()).getTime() + 1500); + setTimeout(() => { + setTaskQueued(0); + }, 1500); + }); + } + + function forgetNow() { + API.backups.forgetNow(backupName).then(() => { + setTaskQueued((new Date()).getTime() + 1500); + setTimeout(() => { + setTaskQueued(0); + }, 1500); + }); + } + + return ( + + {backup.Name}}> + + {/* Left Column */} + +
+ +
+
+ {getChip(backup, snapshots)} +
+
+ + Latest Backup: {snapshots.length > 0 ? new Date(snapshots[0].time).toLocaleDateString() : 'N/A'} + +
+ + + {/* Right Column */} + + {taskQueued < (new Date()).getTime() ? + + } + onClick={fetchBackupDetails} + > + {t('global.refresh')} + + } + onClick={backupNow} + > + Backup Now + + } + onClick={forgetNow} + > + Clean up Now + + : Task has been queued succesfuly} + + Source +
{backup.Source}
+ + Repository +
{backup.Repository}
+ + Backup Schedule +
{crontabToText(backup.Crontab)}
+ + Clean up Schedule +
{crontabToText(backup.CrontabForget)}
+ + + Latest Snapshots +
+ {snapshots.slice(0, 5).map((snapshot, index) => ( + + {new Date(snapshot.time).toLocaleString()} {isMobile ? '' : ` - (${snapshot.short_id || 'Unknown'})`}} + style={{ margin: '5px', width: '100%' }} + /> + + + + + ))} + {snapshots.length === 0 && ( +
No snapshots available
+ )} +
+ + + + + + ); +} \ No newline at end of file diff --git a/client/src/pages/backups/repositories.jsx b/client/src/pages/backups/repositories.jsx new file mode 100644 index 0000000..dfa67bc --- /dev/null +++ b/client/src/pages/backups/repositories.jsx @@ -0,0 +1,89 @@ +import React from "react"; +import { useEffect, useState } from "react"; +import * as API from "../../api"; +import PrettyTableView from "../../components/tableView/prettyTableView"; +import { CloudOutlined, CloudServerOutlined, DeleteOutlined, EditOutlined, FolderOutlined, ReloadOutlined } from "@ant-design/icons"; +import { Checkbox, CircularProgress, ListItemIcon, ListItemText, MenuItem, Stack } from "@mui/material"; +import { crontabToText } from "../../utils/indexs"; +import MenuButton from "../../components/MenuButton"; +import ResponsiveButton from "../../components/responseiveButton"; +import { useTranslation } from "react-i18next"; +import { ConfirmModalDirect } from "../../components/confirmModal"; +import BackupDialog, { BackupDialogInternal } from "./backupDialog"; +import {simplifyNumber} from '../dashboard/components/utils'; + +export const Repositories = () => { + const { t } = useTranslation(); + // const [isAdmin, setIsAdmin] = useState(false); + let isAdmin = true; + const [repositories, setRepositories] = useState(null); + const [loading, setLoading] = useState(false); + const [deleteBackup, setDeleteBackup] = useState(null); + const [editOpened, setEditOpened] = useState(null); + + const refresh = async () => { + setLoading(true); + let configAsync = await API.backups.listRepo(); + setRepositories(configAsync.data || []); + setLoading(false); + }; + + useEffect(() => { + refresh(); + }, []); + + if (loading) { + return ( + + + + ); + } + + return <> + {(repositories) ? <> + + } onClick={refresh}> + {t('global.refresh')} + + + '/cosmos-ui/backups/repo/' + r.id} + data={Object.values(repositories)} + getKey={(r) => r.Name} + columns={[ + { + title: '', + field: () => , + style: { + textAlign: 'right', + width: '64px', + }, + }, + { + title: t('mgmt.backup.repository'), + field: (r) => r.path, + underline: true, + }, + { + title: t('mgmt.backup.repository.size'), + field: (r) => simplifyNumber(r.stats.total_uncompressed_size, "B") + " (" + simplifyNumber(r.stats.total_size, "B") + " on disk)", + underline: true, + }, + { + title: t('mgmt.backup.repository.total_file_count'), + field: (r) => simplifyNumber(r.stats.total_file_count, ""), + underline: true, + }, + { + title: t('mgmt.backup.repository.snapshots_count'), + field: (r) => r.stats.snapshots_count, + underline: true, + }, + ]} + /> + :
+ +
} + ; +}; \ No newline at end of file diff --git a/client/src/pages/backups/restore.jsx b/client/src/pages/backups/restore.jsx new file mode 100644 index 0000000..00ae237 --- /dev/null +++ b/client/src/pages/backups/restore.jsx @@ -0,0 +1,101 @@ +import React, { useEffect, useState } from 'react'; +import { useParams, useSearchParams } from 'react-router-dom'; +import { Stack, CircularProgress, Card, Typography, useMediaQuery, Chip, Button, Select, MenuItem } from '@mui/material'; +import { CloudServerOutlined, CalendarOutlined, FolderOutlined, ReloadOutlined, InfoCircleOutlined, CloudOutlined } from '@ant-design/icons'; +import * as API from '../../api'; +import { crontabToText } from '../../utils/indexs'; +import ResponsiveButton from '../../components/responseiveButton'; +import MainCard from '../../components/MainCard'; +import { useTranslation } from 'react-i18next'; +import FileExplorer from './fileExplorer'; +import BackupFileExplorer from './fileExplorer'; + +export default function BackupRestore({backupName}) { + const [sp] = useSearchParams(); + + const [backup, setBackup] = useState(null); + const [loading, setLoading] = useState(true); + const [snapshots, setSnapshots] = useState([]); + const [selectedSnapshot, setSelectedSnapshot] = useState(sp.get('s')); + const [files, setFiles] = useState([]); + const [isNotFound, setIsNotFound] = useState(false); + const isMobile = useMediaQuery((theme) => theme.breakpoints.down('sm')); + const { t } = useTranslation(); + + const getSubfolderRestoreSize = () => { + API.backups.subfolderRestoreSize(backupName, selectedSnapshot, '/').then((response) => { + console.log(response) + }); + } + + const fetchBackupDetails = async () => { + setLoading(true); + try { + const configResponse = await API.config.get(); + const backupData = configResponse.data.Backup.Backups[backupName]; + + if (!backupData) { + setIsNotFound(true); + return; + } + + setBackup(backupData); + + // Fetch snapshots for this backup + const snapshotsResponse = await API.backups.listSnapshots(backupName); + if(snapshotsResponse.data && snapshotsResponse.data.reverse) { + snapshotsResponse.data.reverse(); + } + + setSnapshots(snapshotsResponse.data || []); + + if(!selectedSnapshot && snapshotsResponse.data.length > 0) { + setSelectedSnapshot(snapshotsResponse.data[0].id); + } + + // getSubfolderRestoreSize(); + } catch (error) { + console.error('Error fetching backup details:', error); + } + setLoading(false); + }; + + useEffect(() => { + fetchBackupDetails(); + }, [backupName]); + + if (loading) { + return ( + + + + ); + } + + if (!backup) { + return ( +
+ + Backup not found: {backupName} + +
+ ); + } + + return ( + +
+ +
+ {"Files in Snapshot"}}> + {selectedSnapshot && API.backups.listFolders(backupName, selectedSnapshot, path)} />} + + + ); +} \ No newline at end of file diff --git a/client/src/pages/backups/restoreDialog.jsx b/client/src/pages/backups/restoreDialog.jsx new file mode 100644 index 0000000..32369ad --- /dev/null +++ b/client/src/pages/backups/restoreDialog.jsx @@ -0,0 +1,127 @@ +import { LoadingButton } from "@mui/lab"; +import { Alert, Grid, Button, Dialog, DialogActions, DialogContent, DialogContentText, DialogTitle, FormHelperText, Stack, TextField } from "@mui/material"; +import React, { useState } from "react"; +import { FormikProvider, useFormik } from "formik"; +import * as yup from "yup"; +import * as API from '../../api'; +import ResponsiveButton from "../../components/responseiveButton"; +import { PlusCircleOutlined, UpOutlined } from "@ant-design/icons"; +import { crontabToText } from "../../utils/indexs"; +import { Trans, useTranslation } from 'react-i18next'; +import { FilePickerButton } from '../../components/filePicker'; +import { CosmosInputText } from "../config/users/formShortcuts"; + +const isAbsolutePath = (path) => path.startsWith('/') || /^[a-zA-Z]:\\/.test(path); // Unix & Windows support + +const RestoreDialogInternal = ({ refresh, open, setOpen, candidatePaths, originalSource, selectedSnapshot, backupName }) => { + const { t } = useTranslation(); + const [done, setDone] = useState(false); + + const formik = useFormik({ + initialValues: { + target: originalSource || '', + }, + validationSchema: yup.object({ + target: yup + .string() + .required(t('global.required')) + .test('is-absolute', t('global.absolutePath'), isAbsolutePath), + }), + onSubmit: async (values, { setErrors, setStatus, setSubmitting }) => { + console.log(values); + setSubmitting(true); + + await API.backups.restoreBackup(backupName, { + snapshotID: selectedSnapshot, + target: values.target, + include: candidatePaths, + }); + + setDone(true); + }, + }); + + return ( + setOpen(false)}> + +
+ + Restore Snapshot + + + + {!done ? + + You are about to restore {candidatePaths.length ? candidatePaths.length : "every"} file(s)/folder(s). This will overwrite existing files. + + + + { + if(path) + formik.setFieldValue('target', path); + }} + size="150%" + select="folder" + /> + + + + {formik.errors.submit && ( + + {formik.errors.submit} + + )} + : + + Restore operation has been queued. You can monitor the progress in the Tasks log on the top right. + + } + + + + + + Restore + + +
+ + + ); +}; + +const RestoreDialog = ({ refresh, candidatePaths, originalSource, selectedSnapshot, backupName }) => { + const { t } = useTranslation(); + const [open, setOpen] = useState(false); + + return ( + <> + {open && } +
+ setOpen(true)} + variant="contained" + size="small" + startIcon={} + > + {candidatePaths.length === 0 ? 'Restore All' : 'Restore Selected'} + +
+ + ); +}; + +export default RestoreDialog; +export { RestoreDialog, RestoreDialogInternal }; \ No newline at end of file diff --git a/client/src/pages/backups/single-backup-index.jsx b/client/src/pages/backups/single-backup-index.jsx new file mode 100644 index 0000000..fe3fcea --- /dev/null +++ b/client/src/pages/backups/single-backup-index.jsx @@ -0,0 +1,54 @@ +import React, { useEffect, useState } from 'react'; +import { useParams } from 'react-router-dom'; +import { Stack, CircularProgress, Card, Typography, useMediaQuery, Chip, Button } from '@mui/material'; +import { CloudServerOutlined, CalendarOutlined, FolderOutlined, ReloadOutlined, InfoCircleOutlined, CloudOutlined } from '@ant-design/icons'; +import * as API from '../../api'; +import { crontabToText } from '../../utils/indexs'; +import ResponsiveButton from '../../components/responseiveButton'; +import MainCard from '../../components/MainCard'; +import { useTranslation } from 'react-i18next'; +import BackupOverview from './overview'; +import PrettyTabbedView from '../../components/tabbedView/tabbedView'; +import Back from '../../components/back'; +import BackupRestore from './restore'; +import EventExplorerStandalone from '../dashboard/eventsExplorerStandalone'; +import SingleRepoIndex from './single-repo-index'; + +export default function SingleBackupIndex() { + const { t } = useTranslation(); + const { backupName } = useParams(); + + return
+ + + +
{backupName}
+ + + + }, + { + title: t('mgmt.backup.restore'), + url: '/restore', + children: + }, + { + title: t('mgmt.backup.snapshots'), + url: '/snapshots', + children: + }, + { + title: t('navigation.monitoring.eventsTitle'), + url: '/events', + children: + }, + ]} /> + +
; +} \ No newline at end of file diff --git a/client/src/pages/backups/single-repo-index.jsx b/client/src/pages/backups/single-repo-index.jsx new file mode 100644 index 0000000..b4088c9 --- /dev/null +++ b/client/src/pages/backups/single-repo-index.jsx @@ -0,0 +1,124 @@ +import React, { useEffect, useState } from 'react'; +import { Link, useParams } from 'react-router-dom'; +import { Stack, CircularProgress, Card, Typography, useMediaQuery, Chip, Button } from '@mui/material'; +import { CloudServerOutlined, CalendarOutlined, FolderOutlined, ReloadOutlined, InfoCircleOutlined, CloudOutlined } from '@ant-design/icons'; +import * as API from '../../api'; +import { crontabToText } from '../../utils/indexs'; +import ResponsiveButton from '../../components/responseiveButton'; +import MainCard from '../../components/MainCard'; +import { useTranslation } from 'react-i18next'; +import PrettyTableView from '../../components/tableView/prettyTableView'; +import { simplifyNumber } from '../dashboard/components/utils'; +import { DeleteButton } from "../../components/delete"; + +export default function SingleRepoIndex({singleBackup}) { + const { backupName } = useParams(); + const [backup, setBackup] = useState(null); + const [loading, setLoading] = useState(true); + const [snapshots, setSnapshots] = useState([]); + const [isNotFound, setIsNotFound] = useState(false); + const isMobile = useMediaQuery((theme) => theme.breakpoints.down('sm')); + const { t } = useTranslation(); + + const infoStyle = { + backgroundColor: 'rgba(0, 0, 0, 0.1)', + padding: '10px', + borderRadius: '5px', + } + + const fetchBackupDetails = async () => { + setLoading(true); + try { + const configResponse = await API.config.get(); + const backupData = configResponse.data.Backup.Backups[backupName]; + + if (!backupData) { + setIsNotFound(true); + return; + } + + setBackup(backupData); + + // Fetch snapshots for this backup + const snapshotsResponse = singleBackup ? + await API.backups.listSnapshots(backupName) : + await API.backups.listSnapshotsFromRepo(backupName) + + if(snapshotsResponse.data && snapshotsResponse.data.reverse) { + snapshotsResponse.data.reverse(); + } + setSnapshots(snapshotsResponse.data || []); + } catch (error) { + console.error('Error fetching backup details:', error); + } + setLoading(false); + }; + + useEffect(() => { + fetchBackupDetails(); + }, [backupName]); + + if (loading) { + return ( + + + + ); + } + + if (!backup) { + return ( +
+ + No repository found for: {backupName} + +
+ ); + } + + return ( + + + } onClick={fetchBackupDetails}> + {t('global.refresh')} + + + r.Name} + columns={[ + { + title: t('mgmt.backup.id'), + field: (r) => r.short_id + ' (' + r.program_version + ')', + }, + { + title: t('mgmt.backup.backup'), + field: (r) => r.tags.join(', '), + }, + { + title: t('mgmt.backup.date'), + field: (r) => (new Date(r.time)).toLocaleString(), + }, + { + title: t('mgmt.backup.data'), + field: (r) => simplifyNumber(r.summary.total_bytes_processed, "B") + ' (' + simplifyNumber(r.summary.data_added, "B") + ' added)', + }, + { + title: t('mgmt.backup.files'), + field: (r) => simplifyNumber(r.summary.total_files_processed, "") + ' (' + simplifyNumber(r.summary.files_new, "") + ' added)', + }, + { + title: "", + field: (r) => { + return + { + await API.backups.forgetSnapshot(backupName, r.id); + }}> + + }, + }, + ]} + /> + + ); +} \ No newline at end of file diff --git a/client/src/pages/config/users/configman.jsx b/client/src/pages/config/users/configman.jsx index 73aee8e..38ba589 100644 --- a/client/src/pages/config/users/configman.jsx +++ b/client/src/pages/config/users/configman.jsx @@ -149,6 +149,7 @@ const ConfigManagement = () => { MonitoringEnabled: !config.MonitoringDisabled, BackupOutputDir: config.BackupOutputDir, + IncrBackupOutputDir: config.Backup.Backups && config.Backup.Backups["Cosmos Internal Backup"] ? config.Backup.Backups["Cosmos Internal Backup"].Repository : "", AdminWhitelistIPs: config.AdminWhitelistIPs && config.AdminWhitelistIPs.join(', '), AdminConstellationOnly: config.AdminConstellationOnly, @@ -170,6 +171,10 @@ const ConfigManagement = () => { onSubmit={async (values, { setErrors, setStatus, setSubmitting }) => { setSubmitting(true); + + if(!values.IncrBackupOutputDir) { + delete config.Backup.Backups["Cosmos Internal Backup"]; + } let toSave = { ...config, @@ -243,6 +248,21 @@ const ConfigManagement = () => { PrimaryColor: values.PrimaryColor, SecondaryColor: values.SecondaryColor }, + Backup: { + ...config.Backup, + Backups: values.IncrBackupOutputDir ? { + ...config.Backup.Backups, + "Cosmos Internal Backup": { + ...config.Backup.Backups["Cosmos Internal Backup"], + "Crontab": "0 0 4 * * *", + "CrontabForget": "0 0 12 * * *", + "Source": status && status.ConfigFolder, + "Password": "", + "Name": "Cosmos Internal Backup", + Repository: values.IncrBackupOutputDir + } + } : config.Backup.Backups + } } return API.config.set(toSave).then((data) => { @@ -415,9 +435,13 @@ const ConfigManagement = () => { + + {t('mgmt.config.general.backupInfo')} + + - { + { if(path) formik.setFieldValue('BackupOutputDir', path); }} size="150%" select="folder" /> @@ -430,6 +454,22 @@ const ConfigManagement = () => { + + + {status && status.Licence && { + if(path) + formik.setFieldValue('IncrBackupOutputDir', path); + }} size="150%" select="folder" />} + + + + {t('mgmt.config.general.logLevelInput')} diff --git a/client/src/pages/config/users/restart.jsx b/client/src/pages/config/users/restart.jsx index c1e2957..6826d7b 100644 --- a/client/src/pages/config/users/restart.jsx +++ b/client/src/pages/config/users/restart.jsx @@ -1,7 +1,7 @@ // material-ui import * as React from 'react'; import { Alert, Button, Stack, Typography } from '@mui/material'; -import { WarningOutlined, PlusCircleOutlined, CopyOutlined, ExclamationCircleOutlined , SyncOutlined, UserOutlined, KeyOutlined } from '@ant-design/icons'; +import { WarningOutlined, PlusCircleOutlined, CopyOutlined, ExclamationCircleOutlined , SyncOutlined, UserOutlined, KeyOutlined, LoadingOutlined } from '@ant-design/icons'; import Table from '@mui/material/Table'; import TableBody from '@mui/material/TableBody'; import TableCell from '@mui/material/TableCell'; @@ -34,6 +34,20 @@ function checkIsOnline() { }); } +const RunningJobs = ({jobs}) => { + return jobs ? (<> + {jobs.length > 0 ? <> + }> + There are jobs running, the server will only restart once all jobs are finished. +
    + {jobs.map((job) =>
  • {job}
    • )} +
+ + + : <>} + ) : <>; +} + const RestartModal = ({openModal, setOpenModal, config, newRoute, isHostMachine }) => { const { t } = useTranslation(); const [isRestarting, setIsRestarting] = useState(false); @@ -44,6 +58,53 @@ const RestartModal = ({openModal, setOpenModal, config, newRoute, isHostMachine let newRouteWarning = config && (config.HTTPConfig.HTTPSCertificateMode == "LETSENCRYPT" && newRoute && (!config.HTTPConfig.DNSChallengeProvider || !config.HTTPConfig.UseWildcardCertificate)) const [errorRestart, setErrorRestart] = useState(null); + const [whenJobsIsEmpty, setWhenJobsIsEmpty] = useState(null); + + const [jobs, setJobs] = useState(null); + + const refresh = () => { + if(!openModal) return; + + API.cron.runningJobs().then((res) => { + setJobs(res.data); + }); + } + + useEffect(() => { + refresh(); + const interval = setInterval(() => { + refresh(); + }, 3000); + return () => clearInterval(interval); + }, []); + + useEffect(() => { + refresh(); + }, [openModal]); + + useEffect(() => { + if (isRestarting && jobs.length == 0) { + if(isHostMachine) { + API.restartServer() + .then((res) => { + }).catch((err) => { + setErrorRestart(err.message); + }); + } else { + API.config.restart() + } + + setTimeout(() => { + if(!errorRestart) { + checkIsOnline(); + } + }, 2500); + + setTimeout(() => { + setWarn(true); + }, 25000) + } + }, [jobs]) return config ? (<> {needsRefresh && <> @@ -78,50 +139,37 @@ const RestartModal = ({openModal, setOpenModal, config, newRoute, isHostMachine } ) :(<> - setOpenModal(false)}> + {setJobs(null); return setOpenModal(false)}}> {!isRestarting ? t('mgmt.config.restart.restartTitle') : t('mgmt.config.restart.restartStatus')} - {errorRestart && }> - {errorRestart} - } - {warn && !errorRestart &&
- }> - {t('mgmt.config.restart.restartTimeoutWarning')}
{t('mgmt.config.restart.restartTimeoutWarningTip')} - -
} - {isRestarting && !errorRestart && -
- -
} - {!isRestarting && t('mgmt.config.restart.restartQuestion')} + + {errorRestart && }> + {errorRestart} + } + {warn && !errorRestart &&
+ }> + {t('mgmt.config.restart.restartTimeoutWarning')}
{t('mgmt.config.restart.restartTimeoutWarningTip')} + +
} +
+ +
+ {isRestarting && !errorRestart && +
+ +
} +
+ {!isRestarting && t('mgmt.config.restart.restartQuestion')} +
+ {!isRestarting && - + } diff --git a/client/src/pages/cron/jobsManage.jsx b/client/src/pages/cron/jobsManage.jsx index 43bc082..562de25 100644 --- a/client/src/pages/cron/jobsManage.jsx +++ b/client/src/pages/cron/jobsManage.jsx @@ -103,6 +103,7 @@ export const CronManager = () => { "Custom": t('mgmt.scheduler.customJobsTitle'), "SnapRAID": t('mgmt.scheduler.parityDiskJobsTitle'), "__OT__SnapRAID": t('mgmt.scheduler.oneTimeJobsTitle'), + "Restic": t('mgmt.scheduler.restic'), }[scheduler])} + element: }, { path: '/cosmos-ui/backups/:backupName/', element: }, + { + path: '/cosmos-ui/backups/repo/:backupName/', + element: + }, { path: '/cosmos-ui/backups/:backupName/:subpath', element: diff --git a/client/src/utils/locales/en/translation.json b/client/src/utils/locales/en/translation.json index 7f1b3e1..ffb0ed9 100644 --- a/client/src/utils/locales/en/translation.json +++ b/client/src/utils/locales/en/translation.json @@ -156,10 +156,12 @@ "mgmt.config.general.forceAutoUpdateButton": "Check for update now", "mgmt.config.general.backupDirInput.backupDirHelperText": "Directory where backups will be stored (relative to the host server `/`)", "mgmt.config.general.backupDirInput.backupDirLabel": "Backup Output Directory (relative to the host server `/`)", + "mgmt.config.general.backupDirInput.incrBackupDirLabel": "Incremental Backup Output Directory (relative to the host server `/`)", "mgmt.config.general.configFileInfo": "This page allow you to edit the configuration file. Any Environment Variable overwritting configuration won't appear here.", "mgmt.config.general.forceMfaCheckbox.forceMfaHelperText": "Require MFA for all users", "mgmt.config.general.forceMfaCheckbox.forceMfaLabel": "Force Multi-Factor Authentication", "mgmt.config.general.logLevelInput": "Level of logging (Default: INFO)", + "mgmt.config.general.backupInfo": "Both those options will allow you to backup your Cosmos config. The first one will simply output your config folder to a zip on each change. The second will create a proper incremental backup (with history) in a folder. The second option requires the backup feature to be unlocked. Both options are optionals", "mgmt.config.general.logLevelInput.logLevelValidation": "Logging Level is required", "mgmt.config.general.mongoDbInput": "MongoDB connection string. It is advised to use Environment variable to store this securely instead. (Optional)", "mgmt.config.general.monitoringCheckbox.monitoringLabel": "Monitoring Enabled", @@ -312,6 +314,7 @@ "mgmt.openid.newMfa.tokenmax6charValidation": "Token must be at most 6 characters", "mgmt.openid.newMfa.tokenmin6charValidation": "Token must be at least 6 characters", "mgmt.openid.newMfa.wrongOtpValidation": "Wrong OTP. Try again", + "mgmt.scheduler.restic": "Backup Jobs", "mgmt.scheduler.customJobsTitle": "Custom Jobs", "mgmt.scheduler.lastLogs": "Last logs for", "mgmt.scheduler.list.action.logs": "Logs", @@ -447,6 +450,9 @@ "mgmt.servapps.viewStackButton": "View Stack", "mgmt.servapps.volumes.list.ScopeTitle": "Scope", "mgmt.servapps.volumes.volumeName": "Volume Name", + "mgmt.storage.newFolder": "New Folder", + "mgmt.storage.newFolderName": "New Folder Name", + "mgmt.storage.newFolderIn": "Creating new folder in", "mgmt.storage.seeFile": "File Browser", "mgmt.storage.pickFile": "Pick a file/folder", "mgmt.storage.selected": "Selected", @@ -859,17 +865,24 @@ "global.name": "Name", "global.password": "Password", "mgmt.backup.sourceTitle": "Source", + "mgmt.backup.backups": "Backups", + "mgmt.backup.snapshots": "Snapshots", + "mgmt.backup.repositories": "Repositories", + "mgmt.backup.repository.size": "Size", + "mgmt.backup.repository.snapshots_count": "# Snapshots", + "mgmt.backup.repository.total_file_count": "# Files", "mgmt.backup.repositoryTitle": "Repository", "mgmt.backup.scheduleTitle": "Schedule", "mgmt.backup.confirmBackupDeletion": "Are you sure you want to delete this backup?", "mgmt.backup.list.restoreText": "Restore", "mgmt.backup.newBackup": "New Backup", "mgmt.backup.createBackup": "Create Backup", - "mgmt.backup.createBackupInfo": "Configure backup settings", + "mgmt.backup.createBackupInfo": "Configure backup settings. All backups are encrypted using AES-256. You can safely re-use the same destination folder (repository) for many backups, the backed up files will be de-dupped, and Cosmos use a tagging system to tell backups appart in the repository. Cosmos uses Restic under the hood, so you can restore your backups even without Cosmos.", "mgmt.backup.min3chars": "Minimum 3 characters required", "mgmt.backup.min8chars": "Minimum 8 characters required", "mgmt.backup.source": "Source", "mgmt.backup.repository": "Repository", "mgmt.backup.schedule": "Crontab Backup Schedule", + "mgmt.backup.restore": "Restore", "global.noData": "No Data To Show" } \ No newline at end of file diff --git a/package.json b/package.json index fd968ce..e08bb64 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cosmos-server", - "version": "0.18.0-unstable4", + "version": "0.18.0-unstable7", "description": "", "main": "test-server.js", "bugs": { diff --git a/rclone.sh b/rclone.sh new file mode 100644 index 0000000..6783f26 --- /dev/null +++ b/rclone.sh @@ -0,0 +1 @@ +./cosmos rclone "$@" \ No newline at end of file diff --git a/src/CRON.go b/src/CRON.go index 31482ae..3adbaf8 100644 --- a/src/CRON.go +++ b/src/CRON.go @@ -12,6 +12,7 @@ import ( "github.com/azukaar/cosmos-server/src/storage" "github.com/azukaar/cosmos-server/src/docker" "github.com/azukaar/cosmos-server/src/proxy" + "github.com/azukaar/cosmos-server/src/cron" "github.com/jasonlvhit/gocron" @@ -157,6 +158,8 @@ func checkUpdatesAvailable() { } } + cron.WaitForAllJobs() // wait for all jobs to finish + utils.Log("Update downloaded, restarting server") storage.StopAllRCloneProcess(true) os.Exit(0) diff --git a/src/backups/API.go b/src/backups/API.go new file mode 100644 index 0000000..b4fcb7f --- /dev/null +++ b/src/backups/API.go @@ -0,0 +1,530 @@ +package backups + +import ( + "fmt" + "encoding/json" + "net/http" + "github.com/gorilla/mux" + "os" + + "github.com/azukaar/cosmos-server/src/utils" +) + +func AddBackupRoute(w http.ResponseWriter, req *http.Request) { + if utils.AdminOnly(w, req) != nil { + return + } + + if req.Method == "POST" { + var request utils.SingleBackupConfig + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + utils.Error("AddBackup: Invalid request", err) + utils.HTTPError(w, "Invalid request: "+err.Error(), http.StatusBadRequest, "BCK001") + return + } + + config := utils.GetMainConfig() + if _, exists := config.Backup.Backups[request.Name]; exists { + utils.HTTPError(w, "Backup already exists", http.StatusBadRequest, "BCK002") + return + } + + // Check repository status + repoInfo, err := os.Stat(request.Repository) + if err != nil && !os.IsNotExist(err) { + utils.Error("AddBackup: Failed to check repository", err) + utils.HTTPError(w, "Failed to check repository: "+err.Error(), http.StatusInternalServerError, "BCK003") + return + } + + var password string + isNewRepo := false + + if os.IsNotExist(err) { + isNewRepo = true + } else if repoInfo.IsDir() { + files, err := os.ReadDir(request.Repository) + if err != nil { + utils.Error("AddBackup: Failed to read repository directory", err) + utils.HTTPError(w, "Failed to read repository directory: "+err.Error(), http.StatusInternalServerError, "BCK005") + return + } + isNewRepo = len(files) == 0 + } + + if isNewRepo { + password = utils.GenerateRandomString(16) + if err := CreateRepository(request.Repository, password); err != nil { + utils.Error("AddBackup: Failed to create repository", err) + utils.HTTPError(w, "Failed to create repository: "+err.Error(), http.StatusInternalServerError, "BCK004") + return + } + } else { + found := false + for _, backup := range config.Backup.Backups { + if backup.Repository == request.Repository { + password = backup.Password + found = true + break + } + } + + if !found { + utils.Error("AddBackup: Cannot find password for existing repository", nil) + utils.HTTPError(w, "Repository exists but password not found", http.StatusBadRequest, "BCK006") + return + } + + if err := CheckRepository(request.Repository, password); err != nil { + utils.Error("AddBackup: Invalid repository", err) + utils.HTTPError(w, "Invalid repository: "+err.Error(), http.StatusInternalServerError, "BCK007") + return + } + } + + request.Password = password + if config.Backup.Backups == nil { + config.Backup.Backups = make(map[string]utils.SingleBackupConfig) + } + config.Backup.Backups[request.Name] = request + utils.SetBaseMainConfig(config) + InitBackups() + + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "OK", + "message": fmt.Sprintf("Added backup %s", request.Name), + }) + } else { + utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001") + } +} + +func EditBackupRoute(w http.ResponseWriter, req *http.Request) { + if utils.AdminOnly(w, req) != nil { + return + } + + if req.Method == "POST" { + var request utils.SingleBackupConfig + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + utils.Error("EditBackup: Invalid request", err) + utils.HTTPError(w, "Invalid request: "+err.Error(), http.StatusBadRequest, "BCK001") + return + } + + config := utils.GetMainConfig() + if _, exists := config.Backup.Backups[request.Name]; !exists { + utils.HTTPError(w, "Backup does not exist", http.StatusBadRequest, "BCK002") + return + } + + current := config.Backup.Backups[request.Name] + + current.Crontab = request.Crontab + current.CrontabForget = request.CrontabForget + current.RetentionPolicy = request.RetentionPolicy + + config.Backup.Backups[request.Name] = current + utils.SetBaseMainConfig(config) + InitBackups() + + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "OK", + "message": fmt.Sprintf("Edited backup %s", request.Name), + }) + } else { + utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001") + } +} + +func RemoveBackupRoute(w http.ResponseWriter, req *http.Request) { + if utils.AdminOnly(w, req) != nil { + return + } + + if req.Method == "DELETE" { + vars := mux.Vars(req) + name := vars["name"] + var request struct { + DeleteRepo bool `json:"deleteRepo"` + } + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + utils.HTTPError(w, "Invalid request: " + err.Error(), http.StatusBadRequest, "BCK001") + return + } + + config := utils.GetMainConfig() + backup, exists := config.Backup.Backups[name] + if !exists { + utils.HTTPError(w, "Backup not found", http.StatusNotFound, "BCK004") + return + } + + // Check if other backups use same repository + otherBackupsUsingRepo := false + for n, b := range config.Backup.Backups { + if n != name && b.Repository == backup.Repository { + otherBackupsUsingRepo = true + break + } + } + + if request.DeleteRepo { + if otherBackupsUsingRepo { + if err := DeleteByTag(backup.Repository, backup.Password, backup.Name); err != nil { + utils.Error("RemoveBackup: Failed to delete snapshots", err) + utils.HTTPError(w, "Failed to delete snapshots: " + err.Error(), http.StatusInternalServerError, "BCK005") + return + } + } else { + if err := DeleteRepository(backup.Repository); err != nil { + utils.Error("RemoveBackup: Failed to delete repository", err) + utils.HTTPError(w, "Failed to delete repository: " + err.Error(), http.StatusInternalServerError, "BCK006") + return + } + } + } else { + if err := DeleteByTag(backup.Repository, backup.Password, backup.Name); err != nil { + utils.Error("RemoveBackup: Failed to delete snapshots", err) + utils.HTTPError(w, "Failed to delete snapshots: " + err.Error(), http.StatusInternalServerError, "BCK005") + return + } + } + + delete(config.Backup.Backups, name) + utils.SetBaseMainConfig(config) + InitBackups() + + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "OK", + "message": fmt.Sprintf("Removed backup %s", name), + }) + } else { + utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001") + } +} + +func ListSnapshotsRoute(w http.ResponseWriter, req *http.Request) { + if utils.AdminOnly(w, req) != nil { + return + } + + if req.Method == "GET" { + vars := mux.Vars(req) + name := vars["name"] + + backup, exists := utils.GetMainConfig().Backup.Backups[name] + if !exists { + utils.HTTPError(w, "Backup not found", http.StatusNotFound, "BCK004") + return + } + + output, err := ListSnapshotsWithFilters(backup.Repository, backup.Password, []string{backup.Name}, "", "") + if err != nil { + utils.Error("ListSnapshots: Failed to list snapshots", err) + utils.HTTPError(w, "Failed to list snapshots: "+err.Error(), http.StatusInternalServerError, "BCK006") + return + } + + w.Header().Set("Content-Type", "application/json") + + var outputJSON []map[string]interface{} + if err := json.Unmarshal([]byte(output), &outputJSON); err != nil { + utils.Error("ListSnapshots: Failed to parse snapshots", err) + utils.HTTPError(w, "Failed to parse snapshots: "+err.Error(), http.StatusInternalServerError, "BCK007") + return + } + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "OK", + "data": outputJSON, + }) + } else { + utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001") + } +} + +func ListFoldersRoute(w http.ResponseWriter, req *http.Request) { + if utils.AdminOnly(w, req) != nil { + return + } + + if req.Method == "GET" { + vars := mux.Vars(req) + name := vars["name"] + snapshot := vars["snapshot"] + path := req.URL.Query().Get("path") + + backup, exists := utils.GetMainConfig().Backup.Backups[name] + if !exists { + utils.HTTPError(w, "Backup not found", http.StatusNotFound, "BCK004") + return + } + + output, err := ListDirectory(backup.Repository, backup.Password, snapshot, path) + if err != nil { + utils.Error("ListFolders: Failed to list folders", err) + utils.HTTPError(w, "Failed to list folders: "+err.Error(), http.StatusInternalServerError, "BCK007") + return + } + + outputF := SplitJSONObjects(output) + + w.Header().Set("Content-Type", "application/json") + var outputJSON []map[string]interface{} + if err := json.Unmarshal([]byte(outputF), &outputJSON); err != nil { + utils.Error("ListSnapshots: Failed to parse snapshots", err) + utils.HTTPError(w, "Failed to parse snapshots: "+err.Error(), http.StatusInternalServerError, "BCK007") + return + } + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "OK", + "data": outputJSON, + }) + } else { + utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001") + } +} + +func RestoreBackupRoute(w http.ResponseWriter, req *http.Request) { + if utils.AdminOnly(w, req) != nil { + return + } + + if req.Method == "POST" { + vars := mux.Vars(req) + name := vars["name"] + + var request struct { + SnapshotID string `json:"snapshotId"` + Target string `json:"target"` + Include []string `json:"include,omitempty"` + } + + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + utils.Error("RestoreBackup: Invalid request", err) + utils.HTTPError(w, "Invalid request: "+err.Error(), http.StatusBadRequest, "BCK008") + return + } + + backup, exists := utils.GetMainConfig().Backup.Backups[name] + if !exists { + utils.HTTPError(w, "Backup not found", http.StatusNotFound, "BCK004") + return + } + + // Verify snapshot belongs to this backup + snapshots, err := ListSnapshotsWithFilters(backup.Repository, backup.Password, []string{backup.Name}, "", "") + if err != nil { + utils.Error("RestoreBackup: Failed to verify snapshot", err) + utils.HTTPError(w, "Failed to verify snapshot: "+err.Error(), http.StatusInternalServerError, "BCK009") + return + } + + var snapshotsArray []map[string]interface{} + if err := json.Unmarshal([]byte(snapshots), &snapshotsArray); err != nil { + utils.Error("RestoreBackup: Failed to parse snapshots", err) + utils.HTTPError(w, "Failed to parse snapshots: "+err.Error(), http.StatusInternalServerError, "BCK010") + return + } + + snapshotFound := false + for _, s := range snapshotsArray { + if s["id"].(string) == request.SnapshotID { + snapshotFound = true + break + } + } + + if !snapshotFound { + utils.HTTPError(w, "Snapshot not found for this backup", http.StatusNotFound, "BCK011") + return + } + + CreateRestoreJob(RestoreConfig{ + Repository: backup.Repository, + Password: backup.Password, + SnapshotID: request.SnapshotID, + Target: request.Target, + Name: backup.Name, + Include: request.Include, + OriginalSource: backup.Source, + }) + + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "OK", + "message": "Restore job created", + }) + } else { + utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001") + } +} + +func ListSnapshotsRouteFromRepo(w http.ResponseWriter, req *http.Request) { + if utils.AdminOnly(w, req) != nil { + return + } + + if req.Method == "GET" { + vars := mux.Vars(req) + name := vars["name"] + + backup, exists := utils.GetMainConfig().Backup.Backups[name] + if !exists { + utils.HTTPError(w, "Backup not found", http.StatusNotFound, "BCK004") + return + } + + output, err := ListSnapshotsWithFilters(backup.Repository, backup.Password, []string{}, "", "") + if err != nil { + utils.Error("ListSnapshots: Failed to list snapshots", err) + utils.HTTPError(w, "Failed to list snapshots: "+err.Error(), http.StatusInternalServerError, "BCK006") + return + } + + w.Header().Set("Content-Type", "application/json") + + var outputJSON []map[string]interface{} + if err := json.Unmarshal([]byte(output), &outputJSON); err != nil { + utils.Error("ListSnapshots: Failed to parse snapshots", err) + utils.HTTPError(w, "Failed to parse snapshots: "+err.Error(), http.StatusInternalServerError, "BCK007") + return + } + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "OK", + "data": outputJSON, + }) + } else { + utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001") + } +} + +func ListRepos(w http.ResponseWriter, req *http.Request) { + if utils.AdminOnly(w, req) != nil { + return + } + + if req.Method == "GET" { + config := utils.GetMainConfig() + repos := map[string]utils.SingleBackupConfig{} + results := map[string]interface{}{} + + for _, backup := range config.Backup.Backups { + found := false + if _, exists := repos[backup.Repository]; exists { + found = true + } + + if !found { + repos[backup.Repository] = backup + + output, err := StatsRepository(backup.Repository, backup.Password) + if err != nil { + utils.Error("ListRepos: Failed to get repository stats", err) + results[backup.Repository] = map[string]interface{}{ + "status": "error", + "id": backup.Name, + "error": err.Error(), + } + } else { + var outputJSON map[string]interface{} + if err := json.Unmarshal([]byte(output), &outputJSON); err != nil { + utils.Error("ListRepos: Failed to parse repository stats", err) + results[backup.Repository] = map[string]interface{}{ + "status": "error", + "id": backup.Name, + "error": err.Error(), + } + } else { + results[backup.Repository] = map[string]interface{}{ + "status": "ok", + "id": backup.Name, + "stats": outputJSON, + "path": backup.Repository, + } + } + } + } + } + + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "OK", + "data": results, + }) + } else { + utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001") + } +} + +func ForgetSnapshotRoute(w http.ResponseWriter, req *http.Request) { + if utils.AdminOnly(w, req) != nil { + return + } + + if req.Method == "DELETE" { + vars := mux.Vars(req) + name := vars["name"] + snapshot := vars["snapshot"] + + config := utils.GetMainConfig() + backup, exists := config.Backup.Backups[name] + if !exists { + utils.HTTPError(w, "Backup not found", http.StatusNotFound, "BCK004") + return + } + + err := ForgetSnapshot(backup.Repository, backup.Password, snapshot) + if err != nil { + utils.Error("ForgetSnapshotRoute: Failed to forget snapshot", err) + utils.HTTPError(w, "Failed to forget snapshot: "+err.Error(), http.StatusInternalServerError, "BCK008") + return + } + + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "OK", + "message": fmt.Sprintf("Forgot snapshot %s from repository %s", snapshot, backup.Repository), + }) + } else { + utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001") + } +} + +func StatsRepositorySubfolderRoute(w http.ResponseWriter, req *http.Request) { + if utils.AdminOnly(w, req) != nil { + return + } + + if req.Method == "GET" { + vars := mux.Vars(req) + name := vars["name"] + snapshot := vars["snapshot"] + path := req.URL.Query().Get("path") + + backup, exists := utils.GetMainConfig().Backup.Backups[name] + if !exists { + utils.HTTPError(w, "Backup not found", http.StatusNotFound, "BCK004") + return + } + + output, err := StatsRepositorySubfolder(backup.Repository, backup.Password, snapshot, path) + if err != nil { + utils.Error("StatsRepositorySubfolder: Failed to get repository stats", err) + utils.HTTPError(w, "Failed to get repository stats: "+err.Error(), http.StatusInternalServerError, "BCK009") + return + } + + w.Header().Set("Content-Type", "application/json") + + var outputJSON map[string]interface{} + if err := json.Unmarshal([]byte(output), &outputJSON); err != nil { + utils.Error("StatsRepositorySubfolder: Failed to parse repository stats", err) + utils.HTTPError(w, "Failed to parse repository stats: "+err.Error(), http.StatusInternalServerError, "BCK010") + return + } + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "OK", + "data": outputJSON, + }) + } else { + utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001") + } +} \ No newline at end of file diff --git a/src/backups/index.go b/src/backups/index.go new file mode 100644 index 0000000..bdceaf9 --- /dev/null +++ b/src/backups/index.go @@ -0,0 +1,92 @@ +package backups + +import ( + "github.com/azukaar/cosmos-server/src/cron" + "github.com/azukaar/cosmos-server/src/utils" +) + +func InitBackups() { + config := utils.GetMainConfig() + + if !utils.FBL.LValid { + utils.Warn("InitBackups: No valid licence found, not starting module.") + return + } + + Repositories := config.Backup.Backups + + cron.ResetScheduler("Restic") + + // check if "Cosmos Internal Backup" exists and is missing password + // if so, create it + internalBackup := "" + intBack := utils.SingleBackupConfig{} + password := "" + + for _, repo := range Repositories { + if repo.Name == "Cosmos Internal Backup" && repo.Password == "" { + utils.Warn("[Backup] Cosmos Internal Backup is missing password, getting one.") + internalBackup = repo.Repository + intBack = repo + } + } + + if internalBackup != "" { + created := false + for _, repo := range Repositories { + if repo.Name != "Cosmos Internal Backup" && repo.Repository == internalBackup { + utils.Log("[Backup] Found backup repository password.") + password = repo.Password + created = true + } + } + + if !created { + utils.Log("[Backup] Password not found. Creating one") + password = utils.GenerateRandomString(16) + } + + + config.Backup.Backups["Cosmos Internal Backup"] = utils.SingleBackupConfig{ + Repository: internalBackup, + Password: password, + Source: intBack.Source, + Crontab: intBack.Crontab, + CrontabForget: intBack.CrontabForget, + RetentionPolicy: intBack.RetentionPolicy, + Name: "Cosmos Internal Backup", + } + + utils.SetBaseMainConfig(config) + config = utils.GetMainConfig() + } + + + for _, repo := range Repositories { + err := CheckRepository(repo.Repository, repo.Password) + + if err != nil { + utils.MajorError("Backups destination unavailable", err) + } else { + // create backup job + CreateBackupJob(BackupConfig{ + Repository: repo.Repository, + Password: repo.Password, + Source: repo.Source, + Name: repo.Name, + Tags: []string{repo.Name}, + // Exclude: repo.Exclude, + }, repo.Crontab) + + // create backup job + CreateForgetJob(BackupConfig{ + Repository: repo.Repository, + Password: repo.Password, + Source: repo.Source, + Name: repo.Name, + Tags: []string{repo.Name}, + Retention: repo.RetentionPolicy, + }, repo.CrontabForget) + } + } +} \ No newline at end of file diff --git a/src/backups/restic.go b/src/backups/restic.go new file mode 100644 index 0000000..13fef8c --- /dev/null +++ b/src/backups/restic.go @@ -0,0 +1,461 @@ +package backups + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "regexp" + "strings" + + "github.com/creack/pty" + "github.com/azukaar/cosmos-server/src/utils" + "github.com/azukaar/cosmos-server/src/cron" + "encoding/json" +) + +var stripAnsi = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]|\x1b\]0;[^\a]*\a|\r`) + +// SplitJSONObjects splits a string containing multiple JSON objects, respecting quotes +func SplitJSONObjects(input string) string { + var objects []string + inQuotes := false + escapeNext := false + start := -1 + braceCount := 0 + + for i, char := range input { + if escapeNext { + escapeNext = false + continue + } + + switch char { + case '"': + if !escapeNext { + inQuotes = !inQuotes + } + case '\\': + escapeNext = true + case '{': + if !inQuotes { + if braceCount == 0 { + start = i + } + braceCount++ + } + case '}': + if !inQuotes { + braceCount-- + if braceCount == 0 && start != -1 { + objects = append(objects, input[start:i+1]) + start = -1 + } + } + } + } + + return "[" + strings.Join(objects, ",") + "]" +} + +// ExecRestic executes a restic command with the given arguments and environment variables +func ExecRestic(args []string, env []string) (string, error) { + cmd := exec.Command("./restic", args...) + cmd.Env = append(os.Environ(), env...) + + // Start the command with a pseudo-terminal + f, err := pty.Start(cmd) + if err != nil { + return "", fmt.Errorf("[Restic] failed to start command: %w", err) + } + defer f.Close() + + // Create a buffer to store all output + var output bytes.Buffer + done := make(chan error, 1) + + // Start a goroutine to read the output + go func() { + buf := make([]byte, 1024) + for { + n, err := f.Read(buf) + if err != nil { + done <- err + return + } + output.Write(buf[:n]) + b := string(buf[:n]) + b = stripAnsi.ReplaceAllString(b, "") + b = strings.TrimSuffix(b, "\n") + utils.Debug("[Restic] " + b) + } + }() + + // Wait for the command to finish + err = cmd.Wait() + if err != nil { + return output.String(), fmt.Errorf("[Restic] command failed: %w\nOutput: %s", err, output.String()) + } + + return output.String(), nil +} + +// CreateRepository initializes a new Restic repository +func CreateRepository(repository, password string) error { + args := []string{"init", "--repo", repository} + env := []string{fmt.Sprintf("RESTIC_PASSWORD=%s", password)} + + output, err := ExecRestic(args, env) + if err != nil { + return err + } + + if !strings.Contains(output, "created restic repository") { + return fmt.Errorf("[Restic] failed to create repository: %s", output) + } + return nil +} + +// DeleteRepository removes a Restic repository +func DeleteRepository(repository string) error { + // First, check if the repository exists + if _, err := os.Stat(repository); os.IsNotExist(err) { + return fmt.Errorf("[Restic] repository does not exist: %s", repository) + } + + // Remove the repository directory + err := os.RemoveAll(repository) + if err != nil { + return fmt.Errorf("[Restic] failed to delete repository: %w", err) + } + return nil +} + +// EditRepositoryPassword changes the password of an existing repository +func EditRepositoryPassword(repository, currentPassword, newPassword string) error { + args := []string{"key", "change", "--repo", repository} + env := []string{ + fmt.Sprintf("RESTIC_PASSWORD=%s", currentPassword), + fmt.Sprintf("RESTIC_NEW_PASSWORD=%s", newPassword), + } + + output, err := ExecRestic(args, env) + if err != nil { + return err + } + + if !strings.Contains(output, "changed password") { + return fmt.Errorf("[Restic] failed to change password: %s", output) + } + return nil +} + +// CheckRepository verifies if a repository exists and is valid +func CheckRepository(repository, password string) error { + args := []string{"check", "--repo", repository} + env := []string{fmt.Sprintf("RESTIC_PASSWORD=%s", password)} + + _, err := ExecRestic(args, env) + return err +} + +// CheckRepository verifies if a repository exists and is valid +func StatsRepository(repository, password string) (string,error) { + args := []string{"stats", "--repo", repository, "--json", "--mode", "raw-data"} + env := []string{fmt.Sprintf("RESTIC_PASSWORD=%s", password)} + + output, err := ExecRestic(args, env) + return output,err +} + +// CheckRepository verifies if a repository exists and is valid +func StatsRepositorySubfolder(repository, password, snapshot, path string) (string,error) { + args := []string{"stats", "--mode", "restore-size", "--repo", repository, "--path", path, snapshot, "--json"} + env := []string{fmt.Sprintf("RESTIC_PASSWORD=%s", password)} + + output, err := ExecRestic(args, env) + return output,err +} + +type BackupConfig struct { + Repository string + Password string + Source string + Name string + Tags []string + Exclude []string + Retention string +} + +// CreateBackupJob creates a backup job configuration +func CreateBackupOneTimeJob(config BackupConfig) { + utils.Log("Creating backup job for " + config.Name) + + args := []string{"backup", "--repo", config.Repository, config.Source} + + // Add tags if specified + for _, tag := range config.Tags { + args = append(args, "--tag", tag) + } + + // Add exclude patterns if specified + for _, exclude := range config.Exclude { + args = append(args, "--exclude", exclude) + } + + env := []string{ + fmt.Sprintf("RESTIC_PASSWORD=%s", config.Password), + } + + cron.RunOneTimeJob(cron.ConfigJob{ + Scheduler: "Restic", + Name: fmt.Sprintf("Restic backup %s", config.Name), + Cancellable: true, + Job: cron.JobFromCommandWithEnv(env, "./restic", args...), + Resource: "backup@" + config.Name, + }) +} + +// CreateBackupJob creates a backup job configuration +func CreateBackupJob(config BackupConfig, crontab string) { + utils.Log("Creating backup job for " + config.Name + " with crontab " + crontab) + + args := []string{"backup", "--repo", config.Repository, config.Source} + + // Add tags if specified + for _, tag := range config.Tags { + args = append(args, "--tag", tag) + } + + // Add exclude patterns if specified + for _, exclude := range config.Exclude { + args = append(args, "--exclude", exclude) + } + + env := []string{ + fmt.Sprintf("RESTIC_PASSWORD=%s", config.Password), + } + + cron.RegisterJob(cron.ConfigJob{ + Scheduler: "Restic", + Name: fmt.Sprintf("Restic backup %s", config.Name), + Cancellable: true, + Job: cron.JobFromCommandWithEnv(env, "./restic", args...), + Crontab: crontab, + Resource: "backup@" + config.Name, + }) +} + +func CreateForgetJob(config BackupConfig, crontab string) { + utils.Log("Creating forget job for " + config.Name + " with crontab " + crontab) + + args := []string{"forget", "--repo", config.Repository, "--prune"} + + ret := config.Retention + if ret == "" { + ret = "--keep-last 3 --keep-daily 7 --keep-weekly 8 --keep-yearly 3" + } + + retArr := strings.Split(ret, " ") + args = append(args, retArr...) + + // Add tags if specified + for _, tag := range config.Tags { + args = append(args, "--tag", tag) + } + + env := []string{ + fmt.Sprintf("RESTIC_PASSWORD=%s", config.Password), + } + + cron.RegisterJob(cron.ConfigJob{ + Scheduler: "Restic", + Name: fmt.Sprintf("Restic forget %s", config.Name), + Cancellable: true, + Job: cron.JobFromCommandWithEnv(env, "./restic", args...), + Crontab: crontab, + Resource: "backup@" + config.Name, + }) +} + +type RestoreConfig struct { + Repository string + Password string + SnapshotID string + Target string + Name string + Include []string + OriginalSource string +} + +// CreateRestoreJob creates a restore job configuration +func CreateRestoreJob(config RestoreConfig) { + args := []string{ + "restore", + "--repo", config.Repository, + config.SnapshotID + ":/" + config.OriginalSource, + "--target", config.Target, + } + + // Add include patterns if specified + for _, include := range config.Include { + //remove OriginalSource from include path + include := strings.TrimPrefix(include, config.OriginalSource) + utils.Debug("[RESTIC] Restore includes: " + include) + args = append(args, "--include", include) + } + + env := []string{ + fmt.Sprintf("RESTIC_PASSWORD=%s", config.Password), + } + + go (func() { + cron.RunOneTimeJob(cron.ConfigJob{ + Scheduler: "Restic", + Name: fmt.Sprintf("Restic restore %s", config.Name), + Cancellable: true, + Job: cron.JobFromCommandWithEnv(env, "./restic", args...), + Resource: "backup@" + config.Name, + }) + })() +} + +// ListSnapshots returns a list of all snapshots in the repository +func ListSnapshots(repository, password string) (string, error) { + args := []string{ + "snapshots", + "--repo", repository, + "--json", // Use JSON format for easier parsing if needed + } + env := []string{fmt.Sprintf("RESTIC_PASSWORD=%s", password)} + + output, err := ExecRestic(args, env) + if err != nil { + return "", fmt.Errorf("[Restic] failed to list snapshots: %w", err) + } + + return output, nil +} + +// ListSnapshotsWithFilters returns a filtered list of snapshots +func ListSnapshotsWithFilters(repository, password string, tags []string, host string, path string) (string, error) { + args := []string{ + "snapshots", + "--repo", repository, + "--json", + } + + // Add optional filters + for _, tag := range tags { + args = append(args, "--tag", tag) + } + if host != "" { + args = append(args, "--host", host) + } + if path != "" { + args = append(args, "--path", path) + } + + env := []string{fmt.Sprintf("RESTIC_PASSWORD=%s", password)} + + output, err := ExecRestic(args, env) + if err != nil { + return "", fmt.Errorf("[Restic] failed to list filtered snapshots: %w", err) + } + + return output, nil +} + +// ListDirectory lists the contents of a directory in a specific snapshot +func ListDirectory(repository, password, snapshotID, path string) (string, error) { + args := []string{ + "ls", + "--repo", repository, + snapshotID, + path, + "--json", // Use JSON format for easier parsing if needed + } + env := []string{fmt.Sprintf("RESTIC_PASSWORD=%s", password)} + + output, err := ExecRestic(args, env) + if err != nil { + return "", fmt.Errorf("[Restic] failed to list directory contents: %w", err) + } + + return output, nil +} + +// ListDirectoryWithFilters lists directory contents with additional filters +func ListDirectoryWithFilters(repository, password, snapshotID, path string, recursive bool, longFormat bool) (string, error) { + args := []string{ + "ls", + "--repo", repository, + snapshotID, + path, + "--json", + } + + if recursive { + args = append(args, "--recursive") + } + if longFormat { + args = append(args, "--long") + } + + env := []string{fmt.Sprintf("RESTIC_PASSWORD=%s", password)} + + output, err := ExecRestic(args, env) + if err != nil { + return "", fmt.Errorf("[Restic] failed to list directory contents with filters: %w", err) + } + + return output, nil +} + +func DeleteByTag(repository string, password string, tag string) error { + // First get all snapshots with this tag + output, err := ListSnapshotsWithFilters(repository, password, []string{tag}, "", "") + if err != nil { + return fmt.Errorf("[Restic] failed to list snapshots for deletion: %w", err) + } + + // Parse the JSON output to get snapshot IDs + var snapshots []map[string]interface{} + if err := json.Unmarshal([]byte(output), &snapshots); err != nil { + return fmt.Errorf("[Restic] failed to parse snapshots: %w", err) + } + + // Extract all snapshot IDs + var ids []string + for _, snap := range snapshots { + ids = append(ids, snap["id"].(string)) + } + + if len(ids) == 0 { + return nil // No snapshots to delete + } + + // Create forget command with all snapshot IDs + args := append([]string{"forget", "--prune"}, ids...) + env := []string{fmt.Sprintf("RESTIC_PASSWORD=%s", password)} + + // Execute the forget command + _, err = ExecRestic(args, env) + if err != nil { + return fmt.Errorf("[Restic] failed to delete snapshots: %w", err) + } + + return nil +} + +func ForgetSnapshot(repository, password, snapshot string) error { + args := []string{"forget", snapshot, "--prune", "--repo", repository} + env := []string{fmt.Sprintf("RESTIC_PASSWORD=%s", password)} + + _, err := ExecRestic(args, env) + if err != nil { + return fmt.Errorf("[Restic] failed to forget snapshot: %w", err) + } + + return nil +} diff --git a/src/configapi/set.go b/src/configapi/set.go index d3d6250..c59135b 100644 --- a/src/configapi/set.go +++ b/src/configapi/set.go @@ -8,6 +8,7 @@ import ( "github.com/azukaar/cosmos-server/src/constellation" "github.com/azukaar/cosmos-server/src/cron" "github.com/azukaar/cosmos-server/src/storage" + "github.com/azukaar/cosmos-server/src/backups" ) func ConfigApiSet(w http.ResponseWriter, req *http.Request) { @@ -60,6 +61,7 @@ func ConfigApiSet(w http.ResponseWriter, req *http.Request) { utils.RestartHTTPServer() cron.InitJobs() cron.InitScheduler() + backups.InitBackups() })() json.NewEncoder(w).Encode(map[string]interface{}{ diff --git a/src/cron/API.go b/src/cron/API.go index 13db578..b831c64 100644 --- a/src/cron/API.go +++ b/src/cron/API.go @@ -206,6 +206,25 @@ func GetJobRoute(w http.ResponseWriter, req *http.Request) { "data": jobs[scheduler][job], }) + return + } else { + utils.Error("Listjobs: Method not allowed " + req.Method, nil) + utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001") + return + } +} + +func GetRunningJobsRoute(w http.ResponseWriter, req *http.Request) { + if utils.AdminOnly(w, req) != nil { + return + } + + if req.Method == "GET" { + jobs := RunningJobs() + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "OK", + "data": jobs, + }) return } else { utils.Error("Listjobs: Method not allowed " + req.Method, nil) diff --git a/src/cron/index.go b/src/cron/index.go index 476400e..1123791 100644 --- a/src/cron/index.go +++ b/src/cron/index.go @@ -10,6 +10,7 @@ import ( "os/exec" "os" "fmt" + "strings" "github.com/go-co-op/gocron/v2" "github.com/creack/pty" @@ -40,11 +41,13 @@ type ConfigJob struct { CancelFunc context.CancelFunc `json:"-"` Container string Timeout time.Duration - MaxLogs int + MaxLogs int + Resource string } var jobsList = map[string]map[string]ConfigJob{} var wasInit = false +var InternalProcessTracker = NewProcessTracker() func GetJobsList() map[string]map[string]ConfigJob { return getJobsList() @@ -75,6 +78,18 @@ func getJobsList() map[string]map[string]ConfigJob { return jobsList } +func RunningJobs() []string { + runningJobs := []string{} + for _, schedulerList := range jobsList { + for _, job := range schedulerList { + if job.Running { + runningJobs = append(runningJobs, job.Name) + } + } + } + return runningJobs +} + func JobFromCommand(command string, args ...string) func(OnLog func(string), OnFail func(error), OnSuccess func(), ctx context.Context, cancel context.CancelFunc) { return JobFromCommandWithEnv([]string{}, command, args...) } @@ -282,6 +297,7 @@ func jobRunner(schedulerName, jobName string) func(OnLog func(string), OnFail fu defer func() { <-RunningLock }() CRONLock <- true + var job ConfigJob var ok bool @@ -325,8 +341,12 @@ func jobRunner(schedulerName, jobName string) func(OnLog func(string), OnFail fu cancel() }() - triggerJobUpdated("start", job.Name) + triggerJobEvent(job, "started", "CRON job " + job.Name + " started", "info", map[string]interface{}{}) + triggerJobUpdated("start", job.Name) + + InternalProcessTracker.StartProcess() + job.Job(OnLog, OnFail, OnSuccess, ctx, cancel) } } @@ -363,6 +383,12 @@ func jobRunner_OnFail(schedulerName, jobName string) func(err error) { jobsList[job.Scheduler][job.Name] = job triggerJobUpdated("fail", job.Name, err.Error()) utils.MajorError("CRON job " + job.Name + " failed", err) + + InternalProcessTracker.EndProcess() + + triggerJobEvent(job, "fail", "CRON job " + job.Name + " failed", "error", map[string]interface{}{ + "error": err.Error(), + }) } <-CRONLock } @@ -377,12 +403,53 @@ func jobRunner_OnSuccess(schedulerName, jobName string) func() { jobsList[job.Scheduler][job.Name] = job triggerJobUpdated("success", job.Name) utils.Log("CRON job " + job.Name + " finished") + + InternalProcessTracker.EndProcess() + + triggerJobEvent(job, "success", "CRON job " + job.Name + " finished", "success", map[string]interface{}{}) } <-CRONLock } } +func WaitForAllJobs() { + utils.Log("Waiting for " + strconv.Itoa(InternalProcessTracker.count) + " jobs to finish...") + InternalProcessTracker.WaitForZero() +} + +func triggerJobEvent(job ConfigJob, eventId, eventTitle, eventLevel string, extra map[string]interface{}) { + utils.TriggerEvent( + "cosmos.cron." + strings.Replace(job.Scheduler, ".", "_", -1) + "." + strings.Replace(job.Name, ".", "_", -1) + "." + eventId, + eventTitle, + eventLevel, + "job@" + job.Scheduler + "@" + job.Name, + extra, + ) + + if job.Resource != "" { + utils.TriggerEvent( + "cosmos.cron-resource." + strings.Replace(job.Scheduler, ".", "_", -1) + "." + strings.Replace(job.Name, ".", "_", -1) + "." + eventId, + eventTitle, + eventLevel, + job.Resource, + extra, + ) + } + + if job.Container != "" { + utils.TriggerEvent( + "cosmos.cron-container." + strings.Replace(job.Scheduler, ".", "_", -1) + "." + strings.Replace(job.Name, ".", "_", -1) + "." + eventId, + eventTitle, + eventLevel, + "container@" + job.Container, + extra, + ) + } +} + func InitScheduler() { + utils.WaitForAllJobs = WaitForAllJobs + var err error if !wasInit { diff --git a/src/cron/tracker.go b/src/cron/tracker.go new file mode 100644 index 0000000..a3a1d3b --- /dev/null +++ b/src/cron/tracker.go @@ -0,0 +1,42 @@ +package cron + +import ( + "sync" +) + +type ProcessTracker struct { + mu sync.Mutex + count int + zeroCh chan struct{} +} + +func NewProcessTracker() *ProcessTracker { + return &ProcessTracker{ + zeroCh: make(chan struct{}, 1), + } +} + +func (pt *ProcessTracker) StartProcess() { + pt.mu.Lock() + pt.count++ + pt.mu.Unlock() +} + +func (pt *ProcessTracker) EndProcess() { + pt.mu.Lock() + pt.count-- + if pt.count == 0 { + select { + case pt.zeroCh <- struct{}{}: + default: + } + } + pt.mu.Unlock() +} + +func (pt *ProcessTracker) WaitForZero() { + if pt.count == 0 { + return + } + <-pt.zeroCh +} \ No newline at end of file diff --git a/src/httpServer.go b/src/httpServer.go index b3da84b..526a12d 100644 --- a/src/httpServer.go +++ b/src/httpServer.go @@ -545,6 +545,7 @@ func InitServer() *mux.Router { srapiAdmin.HandleFunc("/api/jobs/run", cron.RunJobRoute) srapiAdmin.HandleFunc("/api/jobs/get", cron.GetJobRoute) srapiAdmin.HandleFunc("/api/jobs/delete", cron.DeleteJobRoute) + srapiAdmin.HandleFunc("/api/jobs/running", cron.GetRunningJobsRoute) srapiAdmin.HandleFunc("/api/smart-def", storage.ListSmartDef) srapiAdmin.HandleFunc("/api/disks", storage.ListDisksRoute) @@ -558,12 +559,18 @@ func InitServer() *mux.Router { srapiAdmin.HandleFunc("/api/snapraid/{name}/{action}", storage.SnapRAIDRunRoute) srapiAdmin.HandleFunc("/api/rclone-restart", storage.API_Rclone_remountAll) srapiAdmin.HandleFunc("/api/list-dir", storage.ListDirectoryRoute) + srapiAdmin.HandleFunc("/api/new-dir", storage.CreateFolderRoute) + srapiAdmin.HandleFunc("/api/backups-repository", backups.ListRepos) + srapiAdmin.HandleFunc("/api/backups-repository/{name}/snapshots", backups.ListSnapshotsRouteFromRepo) srapiAdmin.HandleFunc("/api/backups/{name}/snapshots", backups.ListSnapshotsRoute) srapiAdmin.HandleFunc("/api/backups/{name}/{snapshot}/folders", backups.ListFoldersRoute) srapiAdmin.HandleFunc("/api/backups/{name}/restore", backups.RestoreBackupRoute) srapiAdmin.HandleFunc("/api/backups", backups.AddBackupRoute) + srapiAdmin.HandleFunc("/api/backups/edit", backups.EditBackupRoute) srapiAdmin.HandleFunc("/api/backups/{name}", backups.RemoveBackupRoute) + srapiAdmin.HandleFunc("/api/backups/{name}/{snapshot}/forget", backups.ForgetSnapshotRoute) + srapiAdmin.HandleFunc("/api/backups/{name}/{snapshot}/subfolder-restore-size", backups.StatsRepositorySubfolderRoute) // srapiAdmin.HandleFunc("/api/storage/raid", storage.RaidListRoute).Methods("GET") // srapiAdmin.HandleFunc("/api/storage/raid", storage.RaidCreateRoute).Methods("POST") diff --git a/src/index.go b/src/index.go index 8754de1..e905ed2 100644 --- a/src/index.go +++ b/src/index.go @@ -295,7 +295,7 @@ func cosmos() { // Has to be done last, so scheduler does not re-init cron.Init() - backups.InitBackups() + go backups.InitBackups() utils.Log("Starting server...") } diff --git a/src/storage/filepicker.go b/src/storage/filepicker.go index 392059d..59faa6f 100644 --- a/src/storage/filepicker.go +++ b/src/storage/filepicker.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "syscall" "strings" + "os" "github.com/azukaar/cosmos-server/src/utils" ) @@ -154,4 +155,79 @@ func ListDirectory(path string) ([]DirectoryListing, error) { } return listings, nil +} + +func CreateFolderRoute(w http.ResponseWriter, req *http.Request) { + if utils.AdminOnly(w, req) != nil { + return + } + + if req.Method == "POST" { + //config := utils.GetMainConfig() + storage := req.URL.Query().Get("storage") + path := req.URL.Query().Get("path") + folder := req.URL.Query().Get("folder") + + if storage == "" { + } + + if path == "" { + path = "/" + } + + storages, err := ListStorage() + if err != nil { + utils.Error("CreateFolderRoute: Error listing storages: "+err.Error(), nil) + utils.HTTPError(w, "Internal server error", http.StatusInternalServerError, "STO002") + return + } + + var basePath string + if storage == "local" { + basePath = "/" + if utils.IsInsideContainer { + basePath = "/mnt/host/" + } + } else { + found := false + for _, s := range storages { + if s.Name == storage { + basePath = s.Path + found = true + break + } + } + if !found { + utils.Error("CreateFolderRoute: Storage not found: "+storage, nil) + utils.HTTPError(w, "Storage not found", http.StatusNotFound, "STO001") + return + } + } + + fullPath := filepath.Join(basePath, path) + fullPath = filepath.Join(fullPath, folder) + + utils.Log("CreateFolderRoute: Creating folder: "+fullPath) + + err = os.MkdirAll(fullPath, 0700) + if err != nil { + utils.Error("CreateFolderRoute: Error listing directory: "+err.Error(), nil) + utils.HTTPError(w, "Internal server error", http.StatusInternalServerError, "STO003") + return + } + + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "OK", + "data": map[string]interface{}{ + "storage": storage, + "path": path, + "storages": storages, + "created": fullPath, + }, + }) + } else { + utils.Error("CreateFolderRoute: Method not allowed "+req.Method, nil) + utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001") + return + } } \ No newline at end of file diff --git a/src/storage/rclone.go b/src/storage/rclone.go index 562b6ff..aadad27 100644 --- a/src/storage/rclone.go +++ b/src/storage/rclone.go @@ -460,6 +460,7 @@ func getStorageList() ([]RemoteStorage, error) { return nil, fmt.Errorf("error getting config dump: %w", err) } + CachedRemoteStorageList = []StorageInfo{} var result map[string]interface{} if err := json.Unmarshal(response, &result); err != nil { @@ -621,6 +622,8 @@ type StorageRoutes struct { var StorageRoutesList []StorageRoutes func remountAll() { + utils.WaitForAllJobs() + StorageRoutesList = []StorageRoutes{} // Mount remote storages @@ -684,6 +687,8 @@ func API_Rclone_remountAll(w http.ResponseWriter, req *http.Request) { } func InitRemoteStorage() bool { + utils.StopAllRCloneProcess = StopAllRCloneProcess + configLocation := utils.CONFIGFOLDER + "rclone.conf" utils.ProxyRCloneUser = utils.GenerateRandomString(8) utils.ProxyRClonePwd = utils.GenerateRandomString(16) diff --git a/src/system.go b/src/system.go index 9468547..73c0ec0 100644 --- a/src/system.go +++ b/src/system.go @@ -10,6 +10,7 @@ import ( "io" "os/exec" "fmt" + "path/filepath" "golang.org/x/sys/cpu" @@ -56,6 +57,9 @@ func StatusRoute(w http.ResponseWriter, req *http.Request) { licenceNumber = utils.FBL.UserNumber } + absoluteConfigPath := utils.CONFIGFOLDER + absoluteConfigPath, _ = filepath.Abs(utils.CONFIGFOLDER) + json.NewEncoder(w).Encode(map[string]interface{}{ "status": "OK", "data": map[string]interface{}{ @@ -87,6 +91,7 @@ func StatusRoute(w http.ResponseWriter, req *http.Request) { "MonitoringDisabled": utils.GetMainConfig().MonitoringDisabled, "Licence": licenceValid, "LicenceNumber": licenceNumber, + "ConfigFolder": absoluteConfigPath, }, }) } else { @@ -233,7 +238,7 @@ func restartHostMachineRoute(w http.ResponseWriter, req *http.Request) { return } - // restart host machine + utils.WaitForAllJobs() err := restartHostMachine() if err != nil { diff --git a/src/utils/types.go b/src/utils/types.go index 6738f90..d56fa22 100644 --- a/src/utils/types.go +++ b/src/utils/types.go @@ -100,6 +100,7 @@ type Config struct { MonitoringDisabled bool MonitoringAlerts map[string]Alert BackupOutputDir string + IncrBackupOutputDir string DisableHostModeWarning bool AdminWhitelistIPs []string AdminConstellationOnly bool @@ -449,4 +450,6 @@ type SingleBackupConfig struct { Password string Source string Crontab string + CrontabForget string + RetentionPolicy string } \ No newline at end of file diff --git a/src/utils/utils.go b/src/utils/utils.go index 9af5d37..99be4cd 100644 --- a/src/utils/utils.go +++ b/src/utils/utils.go @@ -58,6 +58,8 @@ var RestartHTTPServer = func() {} var GetContainerIPByName func(string) (string, error) var DoesContainerExist func(string) bool var CheckDockerNetworkMode func() string +var WaitForAllJobs func() +var StopAllRCloneProcess func(bool) var ResyncConstellationNodes = func() {} @@ -429,6 +431,8 @@ func SaveConfigTofile(config Config) { func RestartServer() { Log("Restarting server...") + WaitForAllJobs() + StopAllRCloneProcess(false) os.Exit(0) }