From e5ab9389e9858b4bd66c475ef2018ed829937adb Mon Sep 17 00:00:00 2001 From: Dennis Date: Wed, 10 Aug 2022 09:02:03 +0200 Subject: [PATCH] Fix for journald filling /var/log using flush from volatile instead of copy, and removing cache files when nearly full (#3967) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix to flush volatile journald to prevent full /var/log at startup * re-introduce directory / in non-rsync copy, and exclude journal* in this copy Note: this may need some testing if someone doesn't use rsync * Fix for non-rsync copy to Disk, since journal cannot be copied onto itself * Backed up journal files are removed with vacuum, so no need to delete them separately * Make room at boot by removing old logs, in addition to the removal later every 15 minutes * remove old archived journal files modified more than 1 day ago * make sure volatile logging is used, since journald sometimes turns persistent at boot remove unused commented lines in ramlog * Fix check for configured journald volatile logging * revert #3799, the rsync --delete Co-authored-by: dennis laptop Co-authored-by: Igor Pečovnik --- .../common/etc/cron.d/armbian-truncate-logs | 2 +- .../bsp/common/usr/lib/armbian/armbian-ramlog | 39 +++++++++++++------ .../usr/lib/armbian/armbian-truncate-logs | 8 ++++ 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/packages/bsp/common/etc/cron.d/armbian-truncate-logs b/packages/bsp/common/etc/cron.d/armbian-truncate-logs index 91a7f9d0a1..9b59bce16f 100644 --- a/packages/bsp/common/etc/cron.d/armbian-truncate-logs +++ b/packages/bsp/common/etc/cron.d/armbian-truncate-logs @@ -1,4 +1,4 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin */15 * * * * root /usr/lib/armbian/armbian-truncate-logs - +@reboot root /usr/lib/armbian/armbian-truncate-logs diff --git a/packages/bsp/common/usr/lib/armbian/armbian-ramlog b/packages/bsp/common/usr/lib/armbian/armbian-ramlog index ef86bf6904..485e8d6f7f 100755 --- a/packages/bsp/common/usr/lib/armbian/armbian-ramlog +++ b/packages/bsp/common/usr/lib/armbian/armbian-ramlog @@ -16,9 +16,9 @@ ENABLED=false # Never touch anything below here. Only edit /etc/default/armbian-ramlog -HDD_LOG=/var/log.hdd/ -RAM_LOG=/var/log/ -LOG2RAM_LOG="${HDD_LOG}armbian-ramlog.log" +HDD_LOG=/var/log.hdd +RAM_LOG=/var/log +LOG2RAM_LOG="${HDD_LOG}/armbian-ramlog.log" LOG_OUTPUT="tee -a $LOG2RAM_LOG" isSafe () { @@ -50,16 +50,29 @@ syncToDisk () { if [ "$USE_RSYNC" = true ]; then ${NoCache} rsync -aXWv \ --exclude "lost+found" --exclude armbian-ramlog.log \ + --exclude 'journal*' --one-file-system \ --links \ ${XTRA_RSYNC_TO[@]+"${XTRA_RSYNC_TO[@]}"} \ - $RAM_LOG $HDD_LOG 2>&1 | $LOG_OUTPUT - ${NoCache} rsync -aXWv \ - --delete \ - --links \ - ${RAM_LOG}journal/ ${HDD_LOG}journal 2>&1 | $LOG_OUTPUT + $RAM_LOG/ $HDD_LOG/ 2>&1 | $LOG_OUTPUT else + rm -f $RAM_LOG/journal # linked journal cannot copied onto itself, it will be re-created below ${NoCache} cp -rfup $RAM_LOG -T $HDD_LOG 2>&1 | $LOG_OUTPUT fi + if [ ! -L $RAM_LOG/journal ] && [ -d $RAM_LOG/journal ]; then # move persistent journal to disk + systemctl stop systemd-journald # journal folder can be in use + if [ -d $HDD_LOG/journal ]; then # optional backup for diagnosis + mv $HDD_LOG/journal $HDD_LOG/journal-$(date +"%Y%M%d-%T") + fi + mv $RAM_LOG/journal $HDD_LOG/journal + systemctl start systemd-journald + fi + if [ ! -L $RAM_LOG/journal ] && [ -d $HDD_LOG/journal ]; then # link persistent journal if applicable + ln -s ${HDD_LOG}/journal ${RAM_LOG}/ + fi + if [ -z "`grep ^Storage=volatile /etc/systemd/journald.conf`" ] ;then # flush to disk when applicable + journalctl --flush # this will flush to persistent log + journalctl --relinquish-var # restart volatile logging + fi sync / } @@ -72,14 +85,18 @@ syncFromDisk () { if [ "$USE_RSYNC" = true ]; then ${NoCache} rsync -aXWv --delete \ --exclude "lost+found" --exclude armbian-ramlog.log \ - --exclude *.gz --exclude *.xz --exclude='*.[0-9]' \ + --exclude '*.gz' --exclude '*.xz' --exclude='*.[0-9]' \ --links \ + --exclude 'journal*' --one-file-system \ ${XTRA_RSYNC_FROM[@]+"${XTRA_RSYNC_FROM[@]}"} \ - $HDD_LOG $RAM_LOG 2>&1 | $LOG_OUTPUT + $HDD_LOG/ $RAM_LOG/ 2>&1 | $LOG_OUTPUT else - ${NoCache} find $HDD_LOG* -maxdepth 1 -type f -not \( -name '*.[0-9]' -or -name '*.xz*' -or -name '*.gz' \) | xargs cp -ut $RAM_LOG + ${NoCache} find $HDD_LOG/* -maxdepth 1 -type f -not \( -name '*.[0-9]' -or -name '*.xz*' -or -name '*.gz' -or -name 'journal*' \) | xargs cp -ut $RAM_LOG/ fi + if [ ! -L $RAM_LOG/journal ] && [ -d $HDD_LOG/journal ]; then # link persistent journal if applicable + ln -s ${HDD_LOG}/journal ${RAM_LOG}/ + fi sync / } diff --git a/packages/bsp/common/usr/lib/armbian/armbian-truncate-logs b/packages/bsp/common/usr/lib/armbian/armbian-truncate-logs index ec01a785e5..a573e6e716 100755 --- a/packages/bsp/common/usr/lib/armbian/armbian-truncate-logs +++ b/packages/bsp/common/usr/lib/armbian/armbian-truncate-logs @@ -16,8 +16,14 @@ JOURNAL_SIZE=5M # size to shrink systemd-journal [ "$ENABLED" != true ] && exit 0 +if [ -z "`grep ^Storage=volatile /etc/systemd/journald.conf`" -a -z "$(ls -A /run/log/journal)" ] ;then + journalctl --relinquish-var # make sure to do volatile logging until the next flush +fi + logusage=$(df /var/log/ --output=pcent | tail -1 |cut -d "%" -f 1) if [ $logusage -ge $treshold ]; then + # remove journal backup files created by armbian-ramdisk in case of duplicate journals directories + rm -rf /var/log.hdd/journal-* # write to SD /usr/lib/armbian/armbian-ramlog write >/dev/null 2>&1 # rotate logs on "disk" @@ -30,4 +36,6 @@ if [ $logusage -ge $treshold ]; then /usr/bin/find /var/log -name '*.[0-9]' -or -name '*.gz' | xargs -r rm -f # vacuum systemd-journald [ -d /var/log/journal ] && journalctl --quiet --vacuum-size=${JOURNAL_SIZE} + # remove old archived journal files modified more than 1 day ago + [ -d /var/log.hdd/journal ] && find /var/log.hdd/journal -ctime 1 \( -name '*@*' -o -name '*~' \) -delete fi