Merge pull request #1046 from nextcloud/enh/695/additional-directories

allow to back up additional directories

Containers/borgbackup/backupscript.sh

@@ -149,6 +149,54 @@ if [ "$BORG_MODE" = backup ]; then
         exit 1
     fi
 
+    # Back up additional directories of the host
+    if [ "$ADDITIONAL_DIRECTORIES_BACKUP" = 'yes' ]; then
+        if [ -d "/docker_volumes/" ]; then
+            DOCKER_VOLUME_DIRS="$(find /docker_volumes -mindepth 1 -maxdepth 1 -type d)"
+            mapfile -t DOCKER_VOLUME_DIRS <<< "$DOCKER_VOLUME_DIRS"
+            for directory in "${DOCKER_VOLUME_DIRS[@]}"; do
+                if [ -z "$(ls -A "$directory")" ]; then
+                    echo "$directory is empty which is not allowed."
+                    exit 1
+                fi
+            done
+            if ! borg create "${BORG_OPTS[@]}" "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-additional-docker-volumes" "/docker_volumes/"; then
+                echo "Deleting the failed backup archive..."
+                borg delete --stats --progress "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-additional-docker-volumes"
+                echo "Backup of additional docker-volumes failed!"
+                exit 1
+            fi
+
+            if ! borg prune --prefix '*_*-additional-docker-volumes' "${BORG_PRUNE_OPTS[@]}"; then
+                echo "Failed to prune additional docker-volumes archives!"
+                exit 1
+            fi
+        fi
+        if [ -d "/host_mounts/" ]; then
+            EXCLUDED_DIRECTORIES=(home/*/.cache root/.cache var/cache lost+found run var/run dev tmp sys proc)
+            # Exclude borg backup cache
+            EXCLUDED_DIRECTORIES+=(var/lib/docker/volumes/nextcloud_aio_backup_cache/_data)
+            # Exclude target directory
+            if [ -n "$BORGBACKUP_HOST_LOCATION" ] && [ "$BORGBACKUP_HOST_LOCATION" != "nextcloud_aio_backupdir" ]; then
+                EXCLUDED_DIRECTORIES+=("$BORGBACKUP_HOST_LOCATION")
+            fi
+            for directory in "${EXCLUDED_DIRECTORIES[@]}"
+            do
+                EXCLUDE_DIRS+=(--exclude "/host_mounts/$directory/")
+            done
+            if ! borg create "${BORG_OPTS[@]}" "${EXCLUDED_DIRECTORIES[@]}" "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-additional-host-mounts" "/host_mounts/"; then
+                echo "Deleting the failed backup archive..."
+                borg delete --stats --progress "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-additional-host-mounts"
+                echo "Backup of additional host-mounts failed!"
+                exit 1
+            fi
+            if ! borg prune --prefix '*_*-additional-host-mounts' "${BORG_PRUNE_OPTS[@]}"; then
+                echo "Failed to prune additional host-mount archives!"
+                exit 1
+            fi
+        fi
+    fi
+
     # Inform user
     get_expiration_time
     echo "Backup finished successfully on $END_DATE_READABLE ($DURATION_READABLE)"
@@ -195,6 +243,11 @@ if [ "$BORG_MODE" = restore ]; then
     # Save current path
     BORG_LOCATION="$(jq '.borg_backup_host_location' /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)"
 
+    # Save Additional Backup dirs
+    if [ -f "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/additional_backup_directories" ]; then
+        ADDITIONAL_BACKUP_DIRECTORIES="$(cat /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/additional_backup_directories)"
+    fi
+
     # Save current nextcloud datadir
     if grep -q '"nextcloud_datadir":' /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json; then
         NEXTCLOUD_DATADIR="$(jq '.nextcloud_datadir' /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)"
@@ -227,6 +280,13 @@ if [ "$BORG_MODE" = restore ]; then
     CONTENTS="$(jq ".nextcloud_datadir = $NEXTCLOUD_DATADIR" /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)"
     echo -E "${CONTENTS}" > /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json
 
+    # Reset the additional backup directories
+    if [ -n "$ADDITIONAL_BACKUP_DIRECTORIES" ]; then
+        echo "$ADDITIONAL_BACKUP_DIRECTORIES" > "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/additional_backup_directories"
+        chown 33:0 "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/additional_backup_directories"
+        chmod 770 "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/additional_backup_directories"
+    fi
+
     umount /tmp/borg
 
     # Inform user

php/containers.json

@@ -233,7 +233,9 @@
         "BORG_PASSWORD=%BORGBACKUP_PASSWORD%",
         "BORG_MODE=%BORGBACKUP_MODE%",
         "SELECTED_RESTORE_TIME=%SELECTED_RESTORE_TIME%",
-        "BACKUP_RESTORE_PASSWORD=%BACKUP_RESTORE_PASSWORD%"
+        "BACKUP_RESTORE_PASSWORD=%BACKUP_RESTORE_PASSWORD%",
+        "ADDITIONAL_DIRECTORIES_BACKUP=%ADDITIONAL_DIRECTORIES_BACKUP%",
+        "BORGBACKUP_HOST_LOCATION=%BORGBACKUP_HOST_LOCATION%"
       ],
       "volumes": [
         {

php/public/index.php

@@ -104,6 +104,7 @@ $app->get('/containers', function ($request, $response, $args) use ($container)
         'is_backup_section_enabled' => $configurationManager->isBackupSectionEnabled(),
         'is_imaginary_enabled' => $configurationManager->isImaginaryEnabled(),
         'is_fulltextsearch_enabled' => $configurationManager->isFulltextsearchEnabled(),
+        'additional_backup_directories' => $configurationManager->GetAdditionalBackupDirectoriesString(),
     ]);
 })->setName('profile');
 $app->get('/login', function ($request, $response, $args) use ($container) {

php/src/Controller/ConfigurationController.php

@@ -57,6 +57,11 @@ class ConfigurationController
                 $this->configurationManager->DeleteDailyBackupTime();
             }
 
+            if (isset($request->getParsedBody()['additional_backup_directories'])) {
+                $additionalBackupDirectories = $request->getParsedBody()['additional_backup_directories'] ?? '';
+                $this->configurationManager->SetAdditionalBackupDirectories($additionalBackupDirectories);
+            }
+
             if (isset($request->getParsedBody()['delete_timezone'])) {
                 $this->configurationManager->DeleteTimezone();
             }

php/src/Data/ConfigurationManager.php

@@ -584,6 +584,45 @@ class ConfigurationManager
         }
     }
 
+    /**
+     * @throws InvalidSettingConfigurationException
+     */
+    public function SetAdditionalBackupDirectories(string $additionalBackupDirectories) : void {
+        $additionalBackupDirectoriesArray = explode("\n", $additionalBackupDirectories);
+        $validDirectories = '';
+        foreach($additionalBackupDirectoriesArray as $entry) {
+            // Trim all unwanted chars on both sites
+            $entry = trim($entry);
+            if ($entry !== "") {
+                if (!preg_match("#^/[0-1a-zA-Z/-_]$#", $entry) && !preg_match("#^[0-1a-zA-Z_-]$#", $entry)) {
+                    throw new InvalidSettingConfigurationException("You entered unallowed characters! Problematic is " . $entry);
+                }
+                $validDirectories .= rtrim($entry, '/') . PHP_EOL;
+            }
+        }
+
+        if ($validDirectories === '') {
+            unlink(DataConst::GetAdditionalBackupDirectoriesFile());
+        } else {
+            file_put_contents(DataConst::GetAdditionalBackupDirectoriesFile(), $validDirectories);
+        }
+    }
+
+    public function GetAdditionalBackupDirectoriesString() : string {
+        if (!file_exists(DataConst::GetAdditionalBackupDirectoriesFile())) {
+            return '';
+        }
+        $additionalBackupDirectories = file_get_contents(DataConst::GetAdditionalBackupDirectoriesFile());
+        return $additionalBackupDirectories;
+    }
+
+    public function GetAdditionalBackupDirectoriesArray() : array {
+        $additionalBackupDirectories = $this->GetAdditionalBackupDirectoriesString();
+        $additionalBackupDirectoriesArray = explode("\n", $additionalBackupDirectories);
+        $additionalBackupDirectoriesArray = array_unique($additionalBackupDirectoriesArray, SORT_REGULAR);
+        return $additionalBackupDirectoriesArray;
+    }
+
     public function isDailyBackupRunning() : bool {
         if (file_exists(DataConst::GetDailyBackupBlockFile())) {
             return true;

php/src/Data/DataConst.php

@@ -31,6 +31,10 @@ class DataConst {
         return self::GetDataDirectory() . '/daily_backup_time';
     }
 
+    public static function GetAdditionalBackupDirectoriesFile() : string {
+        return self::GetDataDirectory() . '/additional_backup_directories';
+    }
+
     public static function GetDailyBackupBlockFile() : string {
         return self::GetDataDirectory() . '/daily_backup_running';
     }

php/src/Docker/DockerActionManager.php

@@ -314,6 +314,14 @@ class DockerActionManager
                     $replacements[1] = $this->configurationManager->GetNextcloudUploadLimit();
                 } elseif ($out[1] === 'NEXTCLOUD_MAX_TIME') {
                     $replacements[1] = $this->configurationManager->GetNextcloudMaxTime();
+                } elseif ($out[1] === 'ADDITIONAL_DIRECTORIES_BACKUP') {
+                    if ($this->configurationManager->GetAdditionalBackupDirectoriesString() !== '') {
+                        $replacements[1] = 'yes';
+                    } else {
+                        $replacements[1] = '';
+                    }
+                } elseif ($out[1] === 'BORGBACKUP_HOST_LOCATION') {
+                    $replacements[1] = $this->configurationManager->GetBorgBackupHostLocation();
                 } else {
                     $replacements[1] = $this->configurationManager->GetSecret($out[1]);
                 }
@@ -354,6 +362,22 @@ class DockerActionManager
             $requestBody['HostConfig']['CapAdd'] = ["SYS_ADMIN"];
             $requestBody['HostConfig']['Devices'] = [["PathOnHost" => "/dev/fuse", "PathInContainer" => "/dev/fuse", "CgroupPermissions" => "rwm"]];
             $requestBody['HostConfig']['SecurityOpt'] = ["apparmor:unconfined"];
+            
+            // Additional backup directories
+            $mounts = [];
+            foreach ($this->configurationManager->GetAdditionalBackupDirectoriesArray() as $additionalBackupDirectories) {
+                if ($additionalBackupDirectories !== '') {
+                    if (!str_starts_with($additionalBackupDirectories, '/')) {
+                        $target = '/docker_volumes/';
+                    } else {
+                        $target = '/host_mounts';
+                    }
+                    $mounts[] = ["Type" => "bind", "Source" => $additionalBackupDirectories, "Target" => $target . $additionalBackupDirectories, "ReadOnly" => true, "BindOptions" => ["NonRecursive" => true]];
+                }
+            }
+            if(count($mounts) > 0) {
+                $requestBody['HostConfig']['Mounts'] = $mounts;
+            }
         }
 
         $url = $this->BuildApiUrl('containers/create?name=' . $container->GetIdentifier());

php/templates/containers.twig

@@ -371,7 +371,7 @@
                                         </form>
 
                                         <h3>Backup restore</h3>
-                                        Choose the backup that you want to restore and click on the button below to restore the selected backup. This will overwrite all your files with the state of the backup so you should consider creating a backup first. It also makes sense to run an integrity check before restoring your files but is not mandatory since it shouldn't be needed in most situations.<br><br>
+                                        Choose the backup that you want to restore and click on the button below to restore the selected backup. This will overwrite all your files with the state of the backup so you should consider creating a backup first. It also makes sense to run an integrity check before restoring your files but is not mandatory since it shouldn't be needed in most situations. Please note that this will not restore additionally chosen backup directories!<br><br>
                                         <form method="POST" action="/api/docker/restore" class="xhr" id="restore_selection">
                                             <input type="hidden" name="{{csrf.keys.name}}" value="{{csrf.name}}">
                                             <input type="hidden" name="{{csrf.keys.value}}" value="{{csrf.value}}">
@@ -406,6 +406,21 @@
                                                 <input class="button" type="submit" value="Disable daily backups" />
                                             </form>
                                         {% endif %}
+
+                                        <h3>Back up additional directories and volumes of your host</h3>
+                                        Below, you can enter directories and docker volumes of your host that will backed up additionally into the same borg backup archive.<br><br>
+                                        <form method="POST" action="/api/configuration" class="xhr">
+                                            <textarea id="additional_backup_directories" name="additional_backup_directories" rows="4" cols="50" placeholder="/directory/on/the/host&#10;my_custom_docker_volume">{{ additional_backup_directories }}</textarea>
+                                            <input type="hidden" name="{{csrf.keys.name}}" value="{{csrf.name}}">
+                                            <input type="hidden" name="{{csrf.keys.value}}" value="{{csrf.value}}">
+                                            <input class="button" type="submit" value="Submit" /><br>
+                                        </form>
+                                        Each line and entry needs to start with a slash or letter/digit. Allowed are only <b>a-z</b>, <b>A-Z</b>, <b>0-9</b>, <b>_</b>, <b>-</b>, and <b>/</b>. If the entry begins with a letter/digit are slashes not supported. Two valid entries are <b>/directory/on/the/host</b> and <b>my_custom_docker_volume</b><br><br/>
+                                        Make sure to specify all storages that you want to back up separately since storages will not be mounted recursively. E.g. providing <b>/</b> as additional backup directory will only back up files and folders that are stored on the root partition and not on the EFI partition or any other. Excluded by the backup will be caches and a few other directories. You should make sure to stop all services before the backup can run correctly if you want to back up the root partition. For automating this see <a href="https://github.com/nextcloud/all-in-one#how-to-stopstartupdate-containers-or-trigger-the-daily-backup-from-a-script-externally">this documentation</a><br><br/>
+                                        Please note that the chosen directories/volumes will not be restored when you restore your instance, so this would need to be done manually. <br><br>
+                                        {% if additional_backup_directories != "" %}
+                                            This option is currently set. You can disable it again by clearing the field and submitting your changes.<br><br>
+                                        {% endif %}
                                     {% endif %}
                                 {% endif %}
                                 {% if has_backup_run_once == false %}

tests/QA/020-backup-and-restore.md

@@ -7,5 +7,6 @@
 - [ ] Submitting a time here should reload the page and reveal at the same place the option to delete the setting again.
 - [ ] When the time of the automatic backup has come (you can test it by choosing a time that is e.g. only a minute away), it should automatically log you out (you can verify by reloading) and after you log in again you should see that the automatic backup is currently running.
 - [ ] After a while you should see that your container are starting and in the Backup and restore section you should see that the backup was successful
+- [ ] When entering additional backup directories, it should allow e.g. `/etc` and `nextcloud_aio_mastercontainer` but not `nextcloud/test`. Running a backup with this should back up these directories/volumes successfully.
 
 You can now continue with [030-aio-password-change.md](./030-aio-password-change.md)