Release 0.2.0

Also log unjoinable instances

CHANGELOG.md

@@ -0,0 +1,41 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+### Changed
+
+### Removed
+
+## [0.2.0] - 2022-04-22
+
+### Added
+
+- Test condition when there are no errors
+- Log unjoinable instances
+- A message when instance can be accessed
+
+### Fixed
+
+- Silence curl's output
+
+## [0.1.1] - 2021-08-29
+
+### Added
+
+- Openssl in `check_command`
+
+### Changed
+
+- Wrapped more lines
+
+## [0.1.0] - 2021-08-29
+
+### Added
+
+- First version

README.md

@@ -44,11 +44,11 @@ and date of expired certificate, default: `1210000`, 2w
 - `INSTANCE_LAST_CHANCE_TIMEOUT`, integer, timeout in seconds to connect to an 
 instance that was previously not accessible, default: `30`
 - `MEDIA_REMOVE_DAYS`, integer, how old in days media attachments have to be 
-before getting removed
+before getting removed, default: `7`
 - `CARDS_REMOVE_DAYS`, integer, how old in days cards previews have to be 
-before getting removed
+before getting removed, default: `15`
 - `STATUSES_REMOVE_DAYS`, integer, how old in days unreferenced statuses have 
-to be before getting removed
+to be before getting removed, default: `30`
 
 Example:
 

tootpaste.sh

@@ -28,11 +28,19 @@ PREV_ERRORS_LOG=/tmp/tootpaste_prev_errors
 
 accounts_cull() {
   
-  $DRY_RUN && $TOOTCTL accounts cull --dry-run --concurrency "$DB_POOL" > "$CULL_LOG"
-  $DRY_RUN || $TOOTCTL accounts cull --concurrency "$DB_POOL" > "$CULL_LOG"
+  $DRY_RUN \
+    && $TOOTCTL accounts cull \
+    --dry-run \
+    --concurrency "$DB_POOL" \
+    > "$CULL_LOG"
+  $DRY_RUN \
+    || $TOOTCTL accounts cull \
+    --concurrency "$DB_POOL" \
+    > "$CULL_LOG"
 
   # Remove instances that have an expired certificate from more than
   # TLS_EXPIRED_MAX_SEC
+  if grep -q 'certificate has expired' "$CULL_LOG"; then
     grep 'certificate has expired' "$CULL_LOG" \
       | awk '{print $NF}' \
       | cut -d'/' -f3 \
@@ -55,23 +63,45 @@ accounts_cull() {
       DATE_DIFF=$(($(date +%s) - TLS_EXPIRED_TS))
       if [[ $DATE_DIFF -gt $TLS_EXPIRED_MAX_SEC ]]; then
         echo "${instance} has a certificate expired for more than TLS_EXPIRED_MAX_SEC, purging..."
-      $DRY_RUN && $TOOTCTL domains purge --concurrency "$DB_POOL" --dry-run "$instance"
-      $DRY_RUN || $TOOTCTL domains purge --concurrency "$DB_POOL" "$instance"
+        $DRY_RUN \
+          && $TOOTCTL domains purge \
+          --concurrency "$DB_POOL" \
+          --dry-run \
+          "$instance"
+        $DRY_RUN \
+          || $TOOTCTL domains purge \
+          --concurrency "$DB_POOL" \
+          "$instance"
       fi
     done < "$TLS_EXPIRED_LOG"
+  fi
 
-  # Log other instances errors, then if they were already in the log, purge
-  # them
+  # Log other instances errors, then if they were already in the log, purge them
+  if grep -q 'https' "$CULL_LOG"; then
     grep \
       -e 'certificate verify failed' \
       -e 'timed out' \
       -e 'sslv3 alert handshake failure' \
       -e 'TooManyRedirectsError' \
+      -e 'EndlessRedirectError' \
+      -e 'HostValidationError' \
       "$CULL_LOG" \
       | awk '{print $NF}' \
       | cut -d'/' -f3 \
       | sort -u \
       > "$OTHER_ERRORS_LOG"
+  fi
+
+  # Log unjoinable instances, then if they were already in the log, purge them
+  if grep -q 'not available during the check:' "$CULL_LOG"; then
+    grep \
+      -A 9999 \
+      'not available during the check:' \
+      "$CULL_LOG" \
+      | tail -n +2 \
+      | sed -E 's/\s+//' \
+      > "$OTHER_ERRORS_LOG"
+  fi
   
   test -f $PREV_ERRORS_LOG || touch $PREV_ERRORS_LOG
   while read -r instance; do
@@ -79,6 +109,7 @@ accounts_cull() {
       error=false
       echo "${instance} was already in error last time your ran tootpaste, trying access..."
       curl \
+        --output /dev/null \
         --silent \
         --show-error \
         --max-time "$INSTANCE_LAST_CHANCE_TIMEOUT" \
@@ -86,8 +117,17 @@ accounts_cull() {
         || error=true
       if $error; then
         echo "${instance} still cannot be accessed, purging..."
-        $DRY_RUN && $TOOTCTL domains purge --concurrency "$DB_POOL" --dry-run "$instance"
-        $DRY_RUN || $TOOTCTL domains purge --concurrency "$DB_POOL" "$instance"
+        $DRY_RUN \
+          && $TOOTCTL domains purge \
+          --concurrency "$DB_POOL" \
+          --dry-run \
+          "$instance"
+        $DRY_RUN \
+          || $TOOTCTL domains purge \
+          --concurrency "$DB_POOL" \
+          "$instance"
+        else
+          echo "${instance} can now be accessed, not purging!"
       fi
     fi
   done < "$OTHER_ERRORS_LOG"
@@ -103,13 +143,30 @@ cache_recount(){
 
 media_remove(){
 
-  $DRY_RUN && $TOOTCTL media remove --days "$MEDIA_REMOVE_DAYS" --concurrency "$DB_POOL" --dry-run
-  $DRY_RUN || $TOOTCTL media remove --days "$MEDIA_REMOVE_DAYS" --concurrency "$DB_POOL"
+  $DRY_RUN \
+    && $TOOTCTL media remove \
+    --days "$MEDIA_REMOVE_DAYS" \
+    --concurrency "$DB_POOL" \
+    --dry-run
+  $DRY_RUN \
+    || $TOOTCTL media remove \
+    --days "$MEDIA_REMOVE_DAYS" \
+    --concurrency "$DB_POOL"
   
   $DRY_RUN && $TOOTCTL media remove-orphans --dry-run
   $DRY_RUN || $TOOTCTL media remove-orphans
-  $DRY_RUN && $TOOTCTL preview_cards remove --days "$MEDIA_REMOVE_DAYS" --concurrency "$DB_POOL" --dry-run
-  $DRY_RUN || $TOOTCTL preview_cards remove --days "$CARDS_REMOVE_DAYS" --concurrency "$DB_POOL" --link
+  
+  $DRY_RUN \
+    && $TOOTCTL preview_cards remove \
+    --days "$MEDIA_REMOVE_DAYS" \
+    --concurrency "$DB_POOL" \
+    --link \
+    --dry-run
+  $DRY_RUN \
+    || $TOOTCTL preview_cards remove \
+    --days "$CARDS_REMOVE_DAYS" \
+    --concurrency "$DB_POOL" \
+    --link
 }
 
 statuses_remove(){
@@ -122,7 +179,7 @@ check_command(){
   command -v "$1" > /dev/null
 }
 
-for command in $TOOTCTL curl grep awk cut sort; do
+for command in $TOOTCTL curl grep awk cut sort openssl; do
   check_command "$command" || (echo "$command not found, exiting..."; exit 1)
 done