From 65a9fe4c95561d09bdac9c52421f0fbc204ab007 Mon Sep 17 00:00:00 2001 From: Simon L Date: Thu, 27 Apr 2023 16:01:16 +0200 Subject: [PATCH 1/2] allow to adjust borgs retention policy Signed-off-by: Simon L --- Containers/borgbackup/Dockerfile | 1 + Containers/borgbackup/backupscript.sh | 10 ++++++---- docker-compose.yml | 1 + php/containers.json | 3 ++- php/src/Data/ConfigurationManager.php | 7 +++++++ php/src/Docker/DockerActionManager.php | 2 ++ php/templates/containers.twig | 2 +- readme.md | 5 +++-- tests/QA/060-environmental-variables.md | 1 + 9 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Containers/borgbackup/Dockerfile b/Containers/borgbackup/Dockerfile index 3aac5df4..6f08740c 100644 --- a/Containers/borgbackup/Dockerfile +++ b/Containers/borgbackup/Dockerfile @@ -21,3 +21,4 @@ RUN chmod +x /usr/bin/start.sh; \ USER root ENTRYPOINT ["start.sh"] LABEL com.centurylinklabs.watchtower.monitor-only="true" +ENV BORG_RETENTION_POLICY="--keep-within=7d --keep-weekly=4 --keep-monthly=6" diff --git a/Containers/borgbackup/backupscript.sh b/Containers/borgbackup/backupscript.sh index 64bb8fd2..8eb64e6d 100644 --- a/Containers/borgbackup/backupscript.sh +++ b/Containers/borgbackup/backupscript.sh @@ -151,11 +151,13 @@ if [ "$BORG_MODE" = backup ]; then rm -f "/nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/skip.update" # Prune options - BORG_PRUNE_OPTS=(--stats --keep-within=7d --keep-weekly=4 --keep-monthly=6 "$BORG_BACKUP_DIRECTORY") + set -x + read -ra BORG_PRUNE_OPTS <<< "$BORG_RETENTION_POLICY" + set +x # Prune archives echo "Pruning the archives..." - if ! borg prune --glob-archives '*_*-nextcloud-aio' "${BORG_PRUNE_OPTS[@]}"; then + if ! borg prune --stats --glob-archives '*_*-nextcloud-aio' "${BORG_PRUNE_OPTS[@]}" "$BORG_BACKUP_DIRECTORY"; then echo "Failed to prune archives!" exit 1 fi @@ -186,7 +188,7 @@ if [ "$BORG_MODE" = backup ]; then exit 1 fi echo "Pruning additional volumes..." - if ! borg prune --glob-archives '*_*-additional-docker-volumes' "${BORG_PRUNE_OPTS[@]}"; then + if ! borg prune --stats --glob-archives '*_*-additional-docker-volumes' "${BORG_PRUNE_OPTS[@]}" "$BORG_BACKUP_DIRECTORY"; then echo "Failed to prune additional docker-volumes archives!" exit 1 fi @@ -216,7 +218,7 @@ if [ "$BORG_MODE" = backup ]; then exit 1 fi echo "Pruning additional host mounts..." - if ! borg prune --glob-archives '*_*-additional-host-mounts' "${BORG_PRUNE_OPTS[@]}"; then + if ! borg prune --stats --glob-archives '*_*-additional-host-mounts' "${BORG_PRUNE_OPTS[@]}" "$BORG_BACKUP_DIRECTORY"; then echo "Failed to prune additional host-mount archives!" exit 1 fi diff --git a/docker-compose.yml b/docker-compose.yml index 3538cc8b..97da2ace 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,7 @@ services: # - AIO_DISABLE_BACKUP_SECTION=false # Setting this to true allows to hide the backup section in the AIO interface. # - APACHE_PORT=11000 # Is needed when running behind a web server or reverse proxy (like Apache, Nginx and else). See https://github.com/nextcloud/all-in-one/blob/main/reverse-proxy.md # - APACHE_IP_BINDING=127.0.0.1 # Should be set when running behind a web server or reverse proxy (like Apache, Nginx and else) that is running on the same host. See https://github.com/nextcloud/all-in-one/blob/main/reverse-proxy.md + # - BORG_RETENTION_POLICY=--keep-within=7d --keep-weekly=4 --keep-monthly=6 # Allows to adjust borgs retention policy. See https://github.com/nextcloud/all-in-one#how-to-adjust-borgs-retention-policy # - COLLABORA_SECCOMP_DISABLED=false # Setting this to true allows to disable Collabora's Seccomp feature. See https://github.com/nextcloud/all-in-one#how-to-disable-collaboras-seccomp-feature # - NEXTCLOUD_DATADIR=/mnt/ncdata # Allows to set the host directory for Nextcloud's datadir. See https://github.com/nextcloud/all-in-one#how-to-change-the-default-location-of-nextclouds-datadir # - NEXTCLOUD_MOUNT=/mnt/ # Allows the Nextcloud container to access the chosen directory on the host. See https://github.com/nextcloud/all-in-one#how-to-allow-the-nextcloud-container-to-access-directories-on-the-host diff --git a/php/containers.json b/php/containers.json index 87cf8125..a9130241 100644 --- a/php/containers.json +++ b/php/containers.json @@ -261,7 +261,8 @@ "BACKUP_RESTORE_PASSWORD=%BACKUP_RESTORE_PASSWORD%", "ADDITIONAL_DIRECTORIES_BACKUP=%ADDITIONAL_DIRECTORIES_BACKUP%", "BORGBACKUP_HOST_LOCATION=%BORGBACKUP_HOST_LOCATION%", - "BORG_HOST_ID=nextcloud-aio-borgbackup" + "BORG_HOST_ID=nextcloud-aio-borgbackup", + "BORG_RETENTION_POLICY=%BORG_RETENTION_POLICY%" ], "volumes": [ { diff --git a/php/src/Data/ConfigurationManager.php b/php/src/Data/ConfigurationManager.php index 205ac7d7..82830d80 100644 --- a/php/src/Data/ConfigurationManager.php +++ b/php/src/Data/ConfigurationManager.php @@ -581,6 +581,13 @@ class ConfigurationManager return $this->GetEnvironmentalVariableOrConfig($envVariableName, $configName, $defaultValue); } + public function GetBorgRetentionPolicy() : string { + $envVariableName = 'BORG_RETENTION_POLICY'; + $configName = 'borg_retention_policy'; + $defaultValue = '--keep-within=7d --keep-weekly=4 --keep-monthly=6'; + return $this->GetEnvironmentalVariableOrConfig($envVariableName, $configName, $defaultValue); + } + public function GetDockerSocketPath() : string { $envVariableName = 'WATCHTOWER_DOCKER_SOCKET_PATH'; $configName = 'docker_socket_path'; diff --git a/php/src/Docker/DockerActionManager.php b/php/src/Docker/DockerActionManager.php index 96767352..cc612532 100644 --- a/php/src/Docker/DockerActionManager.php +++ b/php/src/Docker/DockerActionManager.php @@ -328,6 +328,8 @@ class DockerActionManager $replacements[1] = $this->configurationManager->GetNextcloudMemoryLimit(); } elseif ($out[1] === 'NEXTCLOUD_MAX_TIME') { $replacements[1] = $this->configurationManager->GetNextcloudMaxTime(); + } elseif ($out[1] === 'BORG_RETENTION_POLICY') { + $replacements[1] = $this->configurationManager->GetBorgRetentionPolicy(); } elseif ($out[1] === 'NEXTCLOUD_TRUSTED_CACERTS_DIR') { $replacements[1] = $this->configurationManager->GetTrustedCacertsDir(); } elseif ($out[1] === 'ADDITIONAL_DIRECTORIES_BACKUP') { diff --git a/php/templates/containers.twig b/php/templates/containers.twig index ae0b3c40..54d2d537 100644 --- a/php/templates/containers.twig +++ b/php/templates/containers.twig @@ -407,7 +407,7 @@ The backup itself uses a tool that is called BorgBackup which is a well-known server backup tool that efficiently backs up your files and encrypts them on the fly.

Backups get created in the following directory on the host: {{ borg_backup_host_location }}/borg

Be aware that this solution does not back up files and folders that are mounted into Nextcloud using the external storage app - but you can add further Docker volumes and host paths that you want to back up after the initial backup is done.

- Regarding backup retention, see this.

+ Regarding backup retention, see this.

Daily backups can get enabled after the initial backup is done. Enabling this also allows to enable an option that allows to update all containers, Nextcloud and its apps automatically.

For further documentation and options on this backup solution refer to this section and below.
diff --git a/readme.md b/readme.md index b765af9f..d321dc41 100644 --- a/readme.md +++ b/readme.md @@ -334,10 +334,11 @@ Daily backups can get enabled after the initial backup is done. Enabling this al Be aware that this solution does not back up files and folders that are mounted into Nextcloud using the external storage app - but you can add further Docker volumes and host paths that you want to back up after the initial backup is done. -Regarding backup retention, see [this documentation](https://github.com/nextcloud/all-in-one/discussions/1675). - --- +#### How to adjust borgs retention policy? +The built-in borg-based backup solution has by default a retention policy of `--keep-within=7d --keep-weekly=4 --keep-monthly=6`. See https://borgbackup.readthedocs.io/en/stable/usage/prune.html for what these values mean. You can adjust the upload limit by providing `--env BORG_RETENTION_POLICY="--keep-within=7d --keep-weekly=4 --keep-monthly=6"` to the docker run command of the mastercontainer (but before the last line `nextcloud/all-in-one:latest`! If it was started already, you will need to stop the mastercontainer, remove it (no data will be lost) and recreate it using the docker run command that you initially used) and customize the value to your fitting. ⚠️ Please make sure that this value is valid, otherwise backup pruning will bug out! + #### Are remote borg backups supported? Not directly but you have multiple options to achieve this: diff --git a/tests/QA/060-environmental-variables.md b/tests/QA/060-environmental-variables.md index ea7a375e..5bd7c10b 100644 --- a/tests/QA/060-environmental-variables.md +++ b/tests/QA/060-environmental-variables.md @@ -10,6 +10,7 @@ - [ ] When starting the mastercontainer with `--env NEXTCLOUD_UPLOAD_LIMIT=11G` it should change Nextclouds upload limit to 11G. See https://github.com/nextcloud/all-in-one#how-to-adjust-the-upload-limit-for-nextcloud for allowed values. - [ ] When starting the mastercontainer with `--env NEXTCLOUD_MEMORY_LIMIT=1024M` it should change Nextclouds PHP memory limit to 1024M. See https://github.com/nextcloud/all-in-one#how-to-adjust-the-php-memory-limit-for-nextcloud for allowed values. - [ ] When starting the mastercontainer with `--env NEXTCLOUD_MAX_TIME=4000` it should change Nextclouds upload max time 4000s. See https://github.com/nextcloud/all-in-one#how-to-adjust-the-max-execution-time-for-nextcloud for allowed values. +- [ ] When starting the mastercontainer with `--env BORG_RETENTION_POLICY="--keep-within=1d --keep-weekly=1 --keep-monthly=1"` it should change borgs retention policy to the defined one. This can be checked when creating a backup and looking at the logs. - [ ] When starting the mastercontainer with `--env WATCHTOWER_DOCKER_SOCKET_PATH="$XDG_RUNTIME_DIR/docker.sock"` it should map `$XDG_RUNTIME_DIR/docker.sock` to `/var/run/docker.sock` inside the watchtower container which allow to update the mastercontainer on docker rootless. - [ ] When starting the mastercontainer with `--env AIO_DISABLE_BACKUP_SECTION=true` it should hide the backup section that gets shown after AIO is set up (everything of [020-backup-and-restore](./020-backup-and-restore.md)) and simply show that the backup section is disabled. - [ ] When starting the mastercontainer with `--env NEXTCLOUD_TRUSTED_CACERTS_DIR=/path/to/my/cacerts`, the resulting nextcloud container should trust all the Certification Authorities, whose certificates are included in the directory `/path/to/my/cacerts` on the host. From 8c85e1ef2ecfb4646e03b3461d9b1cf3b5d14e52 Mon Sep 17 00:00:00 2001 From: Simon L Date: Tue, 30 May 2023 11:28:29 +0200 Subject: [PATCH 2/2] adjust wording Signed-off-by: Simon L Signed-off-by: Simon L. --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index d321dc41..69c578d6 100644 --- a/readme.md +++ b/readme.md @@ -337,7 +337,7 @@ Be aware that this solution does not back up files and folders that are mounted --- #### How to adjust borgs retention policy? -The built-in borg-based backup solution has by default a retention policy of `--keep-within=7d --keep-weekly=4 --keep-monthly=6`. See https://borgbackup.readthedocs.io/en/stable/usage/prune.html for what these values mean. You can adjust the upload limit by providing `--env BORG_RETENTION_POLICY="--keep-within=7d --keep-weekly=4 --keep-monthly=6"` to the docker run command of the mastercontainer (but before the last line `nextcloud/all-in-one:latest`! If it was started already, you will need to stop the mastercontainer, remove it (no data will be lost) and recreate it using the docker run command that you initially used) and customize the value to your fitting. ⚠️ Please make sure that this value is valid, otherwise backup pruning will bug out! +The built-in borg-based backup solution has by default a retention policy of `--keep-within=7d --keep-weekly=4 --keep-monthly=6`. See https://borgbackup.readthedocs.io/en/stable/usage/prune.html for what these values mean. You can adjust the retention policy by providing `--env BORG_RETENTION_POLICY="--keep-within=7d --keep-weekly=4 --keep-monthly=6"` to the docker run command of the mastercontainer (but before the last line `nextcloud/all-in-one:latest`! If it was started already, you will need to stop the mastercontainer, remove it (no data will be lost) and recreate it using the docker run command that you initially used) and customize the value to your fitting. ⚠️ Please make sure that this value is valid, otherwise backup pruning will bug out! #### Are remote borg backups supported?