#!/bin/bash

cvmfs_test_name="Offline mode - normal catalog and fixed catalog test"
cvmfs_test_suites="quick"

check_fixed_catalog() {
  echo "* Test offline mode for fixed root catalogs using cvmfs-config.cern.ch"
  cvmfs_mount cvmfs-config.cern.ch "CVMFS_AUTO_UPDATE=false" || return 30

  echo "*** mounting cvmfs, expecting no errors"
  local n_io_err=$(get_xattr nioerr /cvmfs/cvmfs-config.cern.ch)
  local ts_io_err=$(get_xattr timestamp_last_ioerr /cvmfs/cvmfs-config.cern.ch)
  [ x"$n_io_err" = x0 ] || return 31
  [ x"$ts_io_err" = x0 ] || return 32
  local n_offline_msg_before=$(cat_syslog | grep "entering offline mode" | wc -l)
  local n_recovery_msg_before=$(cat_syslog | grep "recovered from offline mode" | wc -l)
  echo "*** number of offline syslog messages: $n_offline_msg_before/$n_recovery_msg_before"
  local host_before=$(get_xattr host /cvmfs/cvmfs-config.cern.ch)
  local proxy_before=$(get_xattr proxy /cvmfs/cvmfs-config.cern.ch)
  echo "*** network connection: $host_before via $proxy_before"

  echo "*** cutting network connectivity, expecting a remount error"
  sudo cvmfs_talk -i cvmfs-config.cern.ch proxy set DIRECT
  sudo cvmfs_talk -i cvmfs-config.cern.ch host set http://127.0.0.1
  sudo cvmfs_talk -i cvmfs-config.cern.ch remount sync
  n_io_err=$(get_xattr nioerr /cvmfs/cvmfs-config.cern.ch)
  ts_io_err=$(get_xattr timestamp_last_ioerr /cvmfs/cvmfs-config.cern.ch)
  [ $n_io_err -eq 0 ] || return 33
  [ $ts_io_err -eq 0 ] || return 34
  n_offline_msg=$(cat_syslog | grep "entering offline mode" | wc -l)
  [ $n_offline_msg -eq $n_offline_msg_before ] || return 35

  echo "*** remount again, expect error number to remain constant"
  sudo cvmfs_talk -i cvmfs-config.cern.ch remount sync
  n_io_err=$(get_xattr nioerr /cvmfs/cvmfs-config.cern.ch)
  [ $n_io_err -eq 0 ] || return 36
  n_offline_msg=$(cat_syslog | grep "entering offline mode" | wc -l)
  [ $n_offline_msg -eq $n_offline_msg_before ] || return 37

  echo "*** network recovery"
  sudo cvmfs_talk -i cvmfs-config.cern.ch proxy set $proxy_before
  sudo cvmfs_talk -i cvmfs-config.cern.ch host set $host_before
  sudo cvmfs_talk -i cvmfs-config.cern.ch remount sync
  n_io_err=$(get_xattr nioerr /cvmfs/cvmfs-config.cern.ch)
  [ $n_io_err -eq 0 ] || return 38
  n_offline_msg=$(cat_syslog | grep "entering offline mode" | wc -l)
  [ $n_offline_msg -eq $n_offline_msg_before ] || return 39
  local n_recovery_msg=$(cat_syslog | grep "recovered from offline mode" | wc -l)
  [ $n_recovery_msg -eq $n_recovery_msg_before ] || return 40

  echo "*** cutting network connectivity again, expecting more remount errors"
  sudo cvmfs_talk -i cvmfs-config.cern.ch proxy set DIRECT
  sudo cvmfs_talk -i cvmfs-config.cern.ch host set http://127.0.0.1
  sudo cvmfs_talk -i cvmfs-config.cern.ch remount sync
  n_io_err=$(get_xattr nioerr /cvmfs/cvmfs-config.cern.ch)
  [ $n_io_err -eq 0 ] || return 41
  n_offline_msg=$(cat_syslog | grep "entering offline mode" | wc -l)
  [ $n_offline_msg -eq $n_offline_msg_before ] || return 42

  echo "* ...done test offline mode for fixed root catalogs using cvmfs-config.cern.ch"
}

check_normal_catalog() {
  echo "* Test offline mode for normal root catalogs using sft.cern.ch"
  cvmfs_mount sft.cern.ch "CVMFS_AUTO_UPDATE=false" || return 10

  echo "*** mounting cvmfs, expecting no errors"
  local n_io_err=$(get_xattr nioerr /cvmfs/sft.cern.ch)
  local ts_io_err=$(get_xattr timestamp_last_ioerr /cvmfs/sft.cern.ch)
  [ x"$n_io_err" = x0 ] || return 11
  [ x"$ts_io_err" = x0 ] || return 12
  local n_offline_msg_before=$(cat_syslog | grep "entering offline mode" | wc -l)
  local n_recovery_msg_before=$(cat_syslog | grep "recovered from offline mode" | wc -l)
  echo "*** number of offline syslog messages: $n_offline_msg_before/$n_recovery_msg_before"
  local host_before=$(get_xattr host /cvmfs/sft.cern.ch)
  local proxy_before=$(get_xattr proxy /cvmfs/sft.cern.ch)
  echo "*** network connection: $host_before via $proxy_before"

  echo "*** cutting network connectivity, expecting a remount error"
  sudo cvmfs_talk -i sft.cern.ch proxy set DIRECT
  sudo cvmfs_talk -i sft.cern.ch host set http://127.0.0.1
  sudo cvmfs_talk -i sft.cern.ch remount sync
  n_io_err=$(get_xattr nioerr /cvmfs/sft.cern.ch)
  ts_io_err=$(get_xattr timestamp_last_ioerr /cvmfs/sft.cern.ch)
  [ $n_io_err -eq 1 ] || return 13
  [ $ts_io_err -gt 0 ] || return 14
  n_offline_msg=$(cat_syslog | grep "entering offline mode" | wc -l)
  [ $n_offline_msg -eq $(($n_offline_msg_before + 1)) ] || return 15

  echo "*** remount again, expect error number to remain constant"
  sudo cvmfs_talk -i sft.cern.ch remount sync
  n_io_err=$(get_xattr nioerr /cvmfs/sft.cern.ch)
  [ $n_io_err -eq 1 ] || return 16
  n_offline_msg=$(cat_syslog | grep "entering offline mode" | wc -l)
  [ $n_offline_msg -eq $(($n_offline_msg_before + 1)) ] || return 17

  echo "*** network recovery"
  sudo cvmfs_talk -i sft.cern.ch proxy set $proxy_before
  sudo cvmfs_talk -i sft.cern.ch host set $host_before
  sudo cvmfs_talk -i sft.cern.ch remount sync
  n_io_err=$(get_xattr nioerr /cvmfs/sft.cern.ch)
  [ $n_io_err -eq 1 ] || return 18
  n_offline_msg=$(cat_syslog | grep "entering offline mode" | wc -l)
  [ $n_offline_msg -eq $(($n_offline_msg_before + 1)) ] || return 19
  local n_recovery_msg=$(cat_syslog | grep "recovered from offline mode" | wc -l)
  [ $n_recovery_msg -eq $(($n_recovery_msg_before + 1)) ] || return 20

  echo "*** cutting network connectivity again, expecting more remount errors"
  sudo cvmfs_talk -i sft.cern.ch proxy set DIRECT
  sudo cvmfs_talk -i sft.cern.ch host set http://127.0.0.1
  sudo cvmfs_talk -i sft.cern.ch remount sync
  n_io_err=$(get_xattr nioerr /cvmfs/sft.cern.ch)
  [ $n_io_err -eq 2 ] || return 21
  n_offline_msg=$(cat_syslog | grep "entering offline mode" | wc -l)
  [ $n_offline_msg -eq $(($n_offline_msg_before + 2)) ] || return 22

  echo "* ...done test offline mode for normal root catalogs using sft.cern.ch"
}

cvmfs_run_test() {
  logfile=$1

  check_normal_catalog || return $?
  echo ""
  check_fixed_catalog || return $?
}

