grep + ssh stops working — centralized log collection across a host fleet

1# you are the loop body
2for host in hosts:                       # O(N) in fleet size
3    out = ssh(host,
4        'grep -h ERROR /var/log/app.log'
5        ' /var/log/app.log.1'
6        ' 2>/dev/null')
7    if ssh.exit_code != 0:
8        print(host, 'unreachable')      # debugging the debugger
9        continue
10    for line in out.splitlines():
11        print(host, line)               # may be empty: file rotated

1# /etc/logrotate.d/app, run nightly by cron
2def rotate(path='/var/log/app.log', keep=3):
3    # shift the chain: app.log.2.gz -> app.log.3.gz, etc.
4    for i in range(keep, 0, -1):
5        mv(f'{path}.{i}.gz', f'{path}.{i+1}.gz')
6    # yesterday's file becomes app.log.1, then gets gzipped
7    mv(path, f'{path}.1')
8    gzip(f'{path}.1')                    # -> app.log.1.gz
9    truncate(path)                       # app starts fresh
10    # anything past `keep` is deleted from local disk
11    rm(f'{path}.{keep+1}.gz')            # silent eviction
12    signal(app_pid, SIGHUP)              # reopen log file

1# the application is the only writer; logrotate is the only mover
2def emit(line):
3    open('/var/log/app.log', 'a').write(line + '\n')
4 
5# on-disk layout after a few days of rotation:
6#   /var/log/app.log         <- today, being appended
7#   /var/log/app.log.1.gz    <- yesterday, compressed
8#   /var/log/app.log.2.gz    <- 2 days ago
9#   /var/log/app.log.3.gz    <- 3 days ago (last kept)
10# anything older was rm'd by Logrotate.rotate — gone from disk.