Release 0.2.0

Also log unjoinable instances

CHANGELOG.md

@@ -0,0 +1,41 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+### Changed
+
+### Removed
+
+## [0.2.0] - 2022-04-22
+
+### Added
+
+- Test condition when there are no errors
+- Log unjoinable instances
+- A message when instance can be accessed
+
+### Fixed
+
+- Silence curl's output
+
+## [0.1.1] - 2021-08-29
+
+### Added
+
+- Openssl in `check_command`
+
+### Changed
+
+- Wrapped more lines
+
+## [0.1.0] - 2021-08-29
+
+### Added
+
+- First version

README.md

@@ -44,11 +44,11 @@ and date of expired certificate, default: `1210000`, 2w
 - `INSTANCE_LAST_CHANCE_TIMEOUT`, integer, timeout in seconds to connect to an 
 instance that was previously not accessible, default: `30`
 - `MEDIA_REMOVE_DAYS`, integer, how old in days media attachments have to be 
-before getting removed
+before getting removed, default: `7`
 - `CARDS_REMOVE_DAYS`, integer, how old in days cards previews have to be 
-before getting removed
+before getting removed, default: `15`
 - `STATUSES_REMOVE_DAYS`, integer, how old in days unreferenced statuses have 
-to be before getting removed
+to be before getting removed, default: `30`
 
 Example:
 

tootpaste.sh

@@ -28,50 +28,80 @@ PREV_ERRORS_LOG=/tmp/tootpaste_prev_errors
 
 accounts_cull() {
   
-  $DRY_RUN && $TOOTCTL accounts cull --dry-run --concurrency "$DB_POOL" > "$CULL_LOG"
+  $DRY_RUN \
-  $DRY_RUN || $TOOTCTL accounts cull --concurrency "$DB_POOL" > "$CULL_LOG"
+    && $TOOTCTL accounts cull \
+    --dry-run \
+    --concurrency "$DB_POOL" \
+    > "$CULL_LOG"
+  $DRY_RUN \
+    || $TOOTCTL accounts cull \
+    --concurrency "$DB_POOL" \
+    > "$CULL_LOG"
 
   # Remove instances that have an expired certificate from more than
   # TLS_EXPIRED_MAX_SEC
-  grep 'certificate has expired' "$CULL_LOG" \
+  if grep -q 'certificate has expired' "$CULL_LOG"; then
-    | awk '{print $NF}' \
+    grep 'certificate has expired' "$CULL_LOG" \
-    | cut -d'/' -f3 \
+      | awk '{print $NF}' \
-    | sort -u \
+      | cut -d'/' -f3 \
-    > "$TLS_EXPIRED_LOG"
+      | sort -u \
+      > "$TLS_EXPIRED_LOG"
   
-  while read -r instance; do
+    while read -r instance; do
-    TLS_EXPIRED_TS=$(
+      TLS_EXPIRED_TS=$(
-      date -d "$(
+        date -d "$(
-        echo Q \
+          echo Q \
-          | openssl s_client \
+            | openssl s_client \
-            -servername "$instance" \
+              -servername "$instance" \
-            -connect "${instance}":443 \
+              -connect "${instance}":443 \
-            2>/dev/null \
+              2>/dev/null \
-          | openssl x509 -noout -dates \
+            | openssl x509 -noout -dates \
-          | grep 'notAfter' \
+            | grep 'notAfter' \
-          | cut -d'=' -f2
+            | cut -d'=' -f2
-      )" +%s
+        )" +%s
-    )
+      )
-    DATE_DIFF=$(($(date +%s) - TLS_EXPIRED_TS))
+      DATE_DIFF=$(($(date +%s) - TLS_EXPIRED_TS))
-    if [[ $DATE_DIFF -gt $TLS_EXPIRED_MAX_SEC ]]; then
+      if [[ $DATE_DIFF -gt $TLS_EXPIRED_MAX_SEC ]]; then
-      echo "${instance} has a certificate expired for more than TLS_EXPIRED_MAX_SEC, purging..."
+        echo "${instance} has a certificate expired for more than TLS_EXPIRED_MAX_SEC, purging..."
-      $DRY_RUN && $TOOTCTL domains purge --concurrency "$DB_POOL" --dry-run "$instance"
+        $DRY_RUN \
-      $DRY_RUN || $TOOTCTL domains purge --concurrency "$DB_POOL" "$instance"
+          && $TOOTCTL domains purge \
-    fi
+          --concurrency "$DB_POOL" \
-  done < "$TLS_EXPIRED_LOG"
+          --dry-run \
+          "$instance"
+        $DRY_RUN \
+          || $TOOTCTL domains purge \
+          --concurrency "$DB_POOL" \
+          "$instance"
+      fi
+    done < "$TLS_EXPIRED_LOG"
+  fi
 
-  # Log other instances errors, then if they were already in the log, purge
+  # Log other instances errors, then if they were already in the log, purge them
-  # them
+  if grep -q 'https' "$CULL_LOG"; then
-  grep \
+    grep \
-    -e 'certificate verify failed' \
+      -e 'certificate verify failed' \
-    -e 'timed out' \
+      -e 'timed out' \
-    -e 'sslv3 alert handshake failure' \
+      -e 'sslv3 alert handshake failure' \
-    -e 'TooManyRedirectsError' \
+      -e 'TooManyRedirectsError' \
-    "$CULL_LOG" \
+      -e 'EndlessRedirectError' \
-    | awk '{print $NF}' \
+      -e 'HostValidationError' \
-    | cut -d'/' -f3 \
+      "$CULL_LOG" \
-    | sort -u \
+      | awk '{print $NF}' \
-    > "$OTHER_ERRORS_LOG"
+      | cut -d'/' -f3 \
+      | sort -u \
+      > "$OTHER_ERRORS_LOG"
+  fi
+
+  # Log unjoinable instances, then if they were already in the log, purge them
+  if grep -q 'not available during the check:' "$CULL_LOG"; then
+    grep \
+      -A 9999 \
+      'not available during the check:' \
+      "$CULL_LOG" \
+      | tail -n +2 \
+      | sed -E 's/\s+//' \
+      > "$OTHER_ERRORS_LOG"
+  fi
   
   test -f $PREV_ERRORS_LOG || touch $PREV_ERRORS_LOG
   while read -r instance; do
@@ -79,6 +109,7 @@ accounts_cull() {
       error=false
       echo "${instance} was already in error last time your ran tootpaste, trying access..."
       curl \
+        --output /dev/null \
         --silent \
         --show-error \
         --max-time "$INSTANCE_LAST_CHANCE_TIMEOUT" \
@@ -86,8 +117,17 @@ accounts_cull() {
         || error=true
       if $error; then
         echo "${instance} still cannot be accessed, purging..."
-        $DRY_RUN && $TOOTCTL domains purge --concurrency "$DB_POOL" --dry-run "$instance"
+        $DRY_RUN \
-        $DRY_RUN || $TOOTCTL domains purge --concurrency "$DB_POOL" "$instance"
+          && $TOOTCTL domains purge \
+          --concurrency "$DB_POOL" \
+          --dry-run \
+          "$instance"
+        $DRY_RUN \
+          || $TOOTCTL domains purge \
+          --concurrency "$DB_POOL" \
+          "$instance"
+        else
+          echo "${instance} can now be accessed, not purging!"
       fi
     fi
   done < "$OTHER_ERRORS_LOG"
@@ -103,13 +143,30 @@ cache_recount(){
 
 media_remove(){
 
-  $DRY_RUN && $TOOTCTL media remove --days "$MEDIA_REMOVE_DAYS" --concurrency "$DB_POOL" --dry-run
+  $DRY_RUN \
-  $DRY_RUN || $TOOTCTL media remove --days "$MEDIA_REMOVE_DAYS" --concurrency "$DB_POOL"
+    && $TOOTCTL media remove \
+    --days "$MEDIA_REMOVE_DAYS" \
+    --concurrency "$DB_POOL" \
+    --dry-run
+  $DRY_RUN \
+    || $TOOTCTL media remove \
+    --days "$MEDIA_REMOVE_DAYS" \
+    --concurrency "$DB_POOL"
   
   $DRY_RUN && $TOOTCTL media remove-orphans --dry-run
   $DRY_RUN || $TOOTCTL media remove-orphans
-  $DRY_RUN && $TOOTCTL preview_cards remove --days "$MEDIA_REMOVE_DAYS" --concurrency "$DB_POOL" --dry-run
+  
-  $DRY_RUN || $TOOTCTL preview_cards remove --days "$CARDS_REMOVE_DAYS" --concurrency "$DB_POOL" --link
+  $DRY_RUN \
+    && $TOOTCTL preview_cards remove \
+    --days "$MEDIA_REMOVE_DAYS" \
+    --concurrency "$DB_POOL" \
+    --link \
+    --dry-run
+  $DRY_RUN \
+    || $TOOTCTL preview_cards remove \
+    --days "$CARDS_REMOVE_DAYS" \
+    --concurrency "$DB_POOL" \
+    --link
 }
 
 statuses_remove(){
@@ -122,7 +179,7 @@ check_command(){
   command -v "$1" > /dev/null
 }
 
-for command in $TOOTCTL curl grep awk cut sort; do
+for command in $TOOTCTL curl grep awk cut sort openssl; do
   check_command "$command" || (echo "$command not found, exiting..."; exit 1)
 done