Monitoring XDCR

[
  {
    "cancelURI": "/controller/cancelXDCR/20763b82bb6b517bd0d15d9f6b78c13c%2Ftravel-sample%2Fdemo",
    "settingsURI": "/settings/replications/20763b82bb6b517bd0d15d9f6b78c13c%2Ftravel-sample%2Fdemo",
    "status": "running",
    "replicationType": "xmem",
    "continuous": true,
    "filterExpression": "",
    "id": "20763b82bb6b517bd0d15d9f6b78c13c/travel-sample/demo",
    "pauseRequested": false,
    "source": "travel-sample",
    "target": "/remoteClusters/20763b82bb6b517bd0d15d9f6b78c13c/buckets/demo",
    "type": "xdcr",
    "recommendedRefreshPeriod": 10,
    "changesLeft": 0,
    "docsChecked": 0,
    "docsWritten": 31591,
    "maxVBReps": null,
    "errors": []
  }
]

curl \
  --user Administrator:password \
  --silent \
  --request GET \
  http://localhost:8091/pools/default/tasks | \
  jq -r 'map(select(.type | contains("xdcr"))) |
    .[] | .id + " (" +.status + ")"'

curl \
  --user Administrator:password \
  --silent \
  --request GET \
  http://localhost:8091/pools/default/tasks | \
  jq -c 'map(select(
    (.type | contains("xdcr"))
    and
    (.status | contains("paused") or contains("notRunning"))
  )) | .[] | .id + " (" +.status + ")"'

curl \
  --user Administrator:password \
  --silent \
  --request GET \
  --data zoom=minute \
  http://localhost:8091/pools/default/buckets/@xdcr-travel-sample/stats | \
  jq -r '.op.samples | to_entries | sort_by(.key) | .[] |
    select(.key | split("/") | length > 1) |
    "  " + (.key) + ": " +
      (.value | add / length | tostring)'

# loop over each of the buckets
for bucket in $(curl \
  --user Administrator:password \
  --silent \
  --request GET \
  http://localhost:8091/pools/default/tasks | \
  jq -r '[ .[] | select(.type == "xdcr") | .source ] | sort | unique | .[]')
do
  echo ""
  echo "Bucket: $bucket"
  echo "================================================================"
  # get the xdcr stats for the bucket
  curl \
    --user Administrator:password \
    --silent \
    --request GET \
    --data zoom=minute \
    http://localhost:8091/pools/default/buckets/@xdcr-$bucket/stats | \
    jq -r '.op.samples | to_entries | sort_by(.key) | .[] |
      select(.key | split("/") | length > 1) |
      "  " + (.key) + ": " +
        (.value | add / length | tostring)'
done

curl \
  --user Administrator:password \
  --silent \
  --request GET \
  --data zoom=minute \
  http://localhost:8091/pools/default/buckets/@xdcr-travel-sample/nodes/172.17.0.2:8091/stats | \
  jq -r '.op.samples | to_entries | sort_by(.key) | .[] |
    select(.key | split("/") | length > 1) |
    "  " + (.key) + ": " +
      (.value | add / length | tostring)'

# loop over each of the buckets
for bucket in $(curl \
  --user Administrator:password \
  --silent \
  --request GET \
  http://localhost:8091/pools/default/tasks | \
  jq -r '[ .[] | select(.type == "xdcr") | .source ] | sort | unique | .[]')
do
  echo ""
  echo "Bucket: $bucket"
  echo "================================================================"
  # get the xdcr stats for the bucket
  curl \
    --user Administrator:password \
    --silent \
    --request GET \
    --data zoom=minute \
    http://localhost:8091/pools/default/buckets/@xdcr-$bucket/nodes/172.17.0.2:8091/stats | \
    jq -r '.op.samples | to_entries | sort_by(.key) | .[] |
      select(.key | split("/") | length > 1) |
      "  " + (.key) + ": " +
        (.value | add / length | tostring)'
done

# get all of the buckets in the cluster that have 1 or more
# xdcr replications configured
buckets=$(curl \
  --user Administrator:password \
  --silent \
  --request GET \
  http://localhost:8091/pools/default/tasks | \
  jq -r '[ .[] | select(.type == "xdcr") | .source ] | sort | unique | .[]')
# get all of the nodes in the cluster running the data service
nodes=$(curl \
  --user Administrator:password \
  --silent \
  --request GET \
  http://localhost:8091/pools/nodes | \
  jq -r '.nodes[] |
    select(.services | contains(["kv"]) == true) |
    .hostname'
)
# loop over each of the buckets
for bucket in ${buckets[@]}
do
  echo ""
  echo "Bucket: $bucket"
  echo "================================================================"
  # loop over each of the nodes in the cluster
  for node in ${nodes[@]}
  do
    echo "Node: $node"
    echo "----------------------------------------------------------------"
    # get the xdcr stats for the bucket on the node
    curl \
      --user Administrator:password \
      --silent \
      --request GET \
      --data zoom=minute \
      http://localhost:8091/pools/default/buckets/@xdcr-$bucket/nodes/$node/stats | \
      jq -r '.op.samples | to_entries | sort_by(.key) | .[] |
        select(.key | split("/") | length > 1) |
        "  " + (.key) + ": " +
          (.value | add / length | tostring)'
    echo ""
  done
done

# set the replication info
REMOTE_CLUSTER='20763b82bb6b517bd0d15d9f6b78c13c'
SOURCE_BUCKET='travel-sample'
target_BUCKET='demo'
STAT_NAME='percent_completeness'

# build the url
STAT_URL="http://localhost:8091/pools/default/buckets/$SOURCE_BUCKET/stats"
STAT_URL="$STAT_URL/replications%2F$REMOTE_CLUSTER%2F$SOURCE_BUCKET"
STAT_URL="$STAT_URL%2F$target_BUCKET%2F$STAT_NAME"

curl \
  --user Administrator:password \
  --silent \
  $STAT_URL | \
  jq -r '.nodeStats | to_entries | .[] |
    (.key | split(":") | .[0]) + ": " + (.value | add / length | tostring)'

curl \
  --user Administrator:password \
  --silent \
  --request GET \
  http://localhost:8091/pools/default/remoteClusters | \
  jq -r '.'

curl \
  --user Administrator:password \
  --silent \
  --request GET \
  --data zoom=minute \
  http://localhost:8091/pools/default/buckets/travel-sample/stats | \
  jq -r '.op.samples |
      "ep_num_ops_get_meta: " + (.ep_num_ops_get_meta | add / length | tostring) +
      "\nep_num_ops_set_meta: " + (.ep_num_ops_set_meta |add / length | tostring) +
      "\nep_num_ops_del_meta: " + (.ep_num_ops_del_meta |add / length | tostring) +
      "\nxdc_ops: " + (.xdc_ops |add / length | tostring)'

curl \
  --user Administrator:password \
  --silent \
  --request GET \
  --data zoom=minute \
  http://localhost:8091/pools/default/buckets/travel-sample/stats | \
  jq -r '.op.samples |
    "avg_active_timestamp_drift: " +
      (.avg_active_timestamp_drift | add / length | tostring) +
    "\navg_replica_timestamp_drift: " +
      (.avg_replica_timestamp_drift | add / length | tostring) +
    "\nep_active_hlc_drift: " +
      (.ep_active_hlc_drift | add / length | tostring) +
    "\nep_active_hlc_drift_count: " +
      (.ep_active_hlc_drift_count | add / length | tostring) +
    "\nep_replica_hlc_drift: " +
      (.ep_replica_hlc_drift | add / length | tostring) +
    "\nep_replica_hlc_drift_count: " +
      (.ep_replica_hlc_drift_count | add / length | tostring) +
    "\nep_active_ahead_exceptions: " +
      (.ep_active_ahead_exceptions | add / length | tostring) +
    "\nep_clock_cas_drift_threshold_exceeded: " +
      (.ep_clock_cas_drift_threshold_exceeded | add / length | tostring)'