460 lines
21 KiB
Python
460 lines
21 KiB
Python
"""Bounded Linux/Docker memory observation; GET only, no service changes.
|
|
|
|
Run alongside the named acceptance workload. Output is evidence, never an
|
|
automatic declaration that a 2 GB deployment has passed its load acceptance.
|
|
Requires Python 3 and read access to /proc and the local Docker socket.
|
|
"""
|
|
import argparse
|
|
import http.client
|
|
import json
|
|
import os
|
|
import re
|
|
import socket
|
|
import stat
|
|
import threading
|
|
import time
|
|
|
|
MAX_RESPONSE = 4 * 1024 * 1024
|
|
MAX_FILE_BYTES = 65536
|
|
MAX_FILE_LINES = 256
|
|
CGROUP_ROOT = '/sys/fs/cgroup'
|
|
CGROUP_FILES = ('memory.current', 'memory.peak', 'memory.events',
|
|
'memory.swap.current', 'memory.swap.peak', 'memory.swap.events',
|
|
'memory.max', 'memory.swap.max', 'cpu.stat')
|
|
|
|
|
|
class CgroupUnavailable(ValueError):
|
|
"""Fixed reason codes only; never include private paths or raw content."""
|
|
|
|
|
|
class ObservationDeadlineExceeded(RuntimeError):
|
|
pass
|
|
|
|
|
|
def check_deadline(deadline):
|
|
if deadline is not None and time.monotonic() >= deadline:
|
|
raise ObservationDeadlineExceeded('Observation time bound exceeded')
|
|
|
|
|
|
def read_bounded(path, expected_device=None, deadline=None):
|
|
# Fixed production paths only; no CLI path or executable is accepted.
|
|
check_deadline(deadline)
|
|
descriptor = os.open(path, os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_NONBLOCK', 0))
|
|
with os.fdopen(descriptor, 'rb') as handle:
|
|
identity = os.fstat(handle.fileno())
|
|
if not stat.S_ISREG(identity.st_mode) or (expected_device is not None and identity.st_dev != expected_device):
|
|
raise ValueError('Non-regular observation')
|
|
raw = handle.read(MAX_FILE_BYTES + 1)
|
|
check_deadline(deadline)
|
|
if len(raw) > MAX_FILE_BYTES:
|
|
raise ValueError('Observation file too large')
|
|
text = raw.decode('ascii')
|
|
if len(text.splitlines()) > MAX_FILE_LINES:
|
|
raise ValueError('Too many observation lines')
|
|
return text
|
|
|
|
|
|
def parse_counter(text):
|
|
if not re.fullmatch(r'(?:0|[1-9][0-9]{0,19})\n?', text):
|
|
raise ValueError('Invalid counter')
|
|
value = int(text)
|
|
if value > 2 ** 64 - 1:
|
|
raise ValueError('Counter overflow')
|
|
return value
|
|
|
|
|
|
def parse_events(text, required):
|
|
lines = text.splitlines()
|
|
if not 1 <= len(lines) <= 32:
|
|
raise ValueError('Invalid event count')
|
|
result = {}
|
|
for line in lines:
|
|
fields = line.split(' ')
|
|
if len(fields) != 2 or not re.fullmatch(r'[a-z_]{1,64}', fields[0]) or fields[0] in result:
|
|
raise ValueError('Invalid event')
|
|
result[fields[0]] = parse_counter(fields[1])
|
|
if not set(required).issubset(result):
|
|
raise ValueError('Incomplete events')
|
|
return result
|
|
|
|
|
|
|
|
def parse_limit(text):
|
|
# Preserve unlimited vs zero vs unavailable; never substitute host RAM.
|
|
return 'max' if text in ('max', 'max\n') else parse_counter(text)
|
|
|
|
|
|
def parse_cpu_throttling(text):
|
|
required = ('nr_periods', 'nr_throttled', 'throttled_usec')
|
|
parsed = parse_events(text, required)
|
|
# Kernel extensions are validated but do not expand the output whitelist.
|
|
return {key: parsed[key] for key in required}
|
|
|
|
|
|
def parse_memory_pressure(text):
|
|
rows = text.splitlines()
|
|
if len(rows) != 2:
|
|
raise ValueError('Incomplete memory pressure')
|
|
result = {}
|
|
for line in rows:
|
|
match = re.fullmatch(r'(some|full) avg10=([0-9]{1,3}\.[0-9]{2}) avg60=([0-9]{1,3}\.[0-9]{2}) avg300=([0-9]{1,3}\.[0-9]{2}) total=((?:0|[1-9][0-9]{0,19}))', line)
|
|
if not match or match[1] in result:
|
|
raise ValueError('Invalid memory pressure')
|
|
for value in match.group(2, 3, 4):
|
|
whole, fraction = value.split('.')
|
|
if int(whole) * 100 + int(fraction) > 10000:
|
|
raise ValueError('Invalid memory pressure percentage')
|
|
result[match[1]] = {'avg10Percent': match[2], 'avg60Percent': match[3],
|
|
'avg300Percent': match[4], 'totalMicros': parse_counter(match[5])}
|
|
if set(result) != {'some', 'full'}:
|
|
raise ValueError('Incomplete memory pressure')
|
|
return result
|
|
|
|
|
|
def configured_memory_swap(inspect):
|
|
# Docker MemorySwap is TOTAL memory+swap; 0 means daemon default, -1 unlimited.
|
|
host = inspect.get('HostConfig')
|
|
value = host.get('MemorySwap') if isinstance(host, dict) else None
|
|
if type(value) is not int or not -1 <= value <= 2 ** 63 - 1:
|
|
return None
|
|
return value
|
|
|
|
|
|
def resource_limits_stable(before, after):
|
|
values = []
|
|
for item in (before, after):
|
|
host = item.get('HostConfig')
|
|
memory = host.get('Memory') if isinstance(host, dict) else None
|
|
swap = configured_memory_swap(item)
|
|
if type(memory) is not int or not 0 <= memory <= 2 ** 63 - 1 or swap is None:
|
|
return None
|
|
values.append((memory, swap))
|
|
return values[0] == values[1]
|
|
|
|
|
|
def collect_host(deadline=None):
|
|
memory = parse_host(read_bounded('/proc/meminfo', deadline=deadline),
|
|
read_bounded('/proc/vmstat', deadline=deadline))
|
|
try:
|
|
pressure = parse_memory_pressure(read_bounded('/proc/pressure/memory', deadline=deadline))
|
|
memory['memoryPressure'] = {'status': 'available', 'metrics': pressure, 'reason': None}
|
|
except (OSError, ValueError, UnicodeError):
|
|
memory['memoryPressure'] = {'status': 'unavailable', 'metrics': None,
|
|
'reason': 'missing-unreadable-or-malformed'}
|
|
return memory
|
|
|
|
|
|
def container_identity(inspect, identifier):
|
|
state = inspect['State']
|
|
if (inspect.get('Id') != identifier or not re.fullmatch(r'[a-f0-9]{64}', identifier)
|
|
or type(state.get('Pid')) is not int or not 0 < state['Pid'] <= 2 ** 31 - 1
|
|
or state.get('Running') is not True or type(state.get('OOMKilled')) is not bool
|
|
or not isinstance(state.get('StartedAt'), str) or not state['StartedAt']
|
|
or type(inspect.get('RestartCount')) is not int or inspect['RestartCount'] < 0
|
|
or not isinstance(inspect.get('Image'), str) or not inspect['Image']):
|
|
raise ValueError('Unavailable container identity')
|
|
return (identifier, inspect['Image'], state['StartedAt'], state['Pid'], inspect['RestartCount'], state['OOMKilled'])
|
|
|
|
|
|
def cgroup_location(pid, identifier, deadline=None):
|
|
check_deadline(deadline)
|
|
mounts = read_bounded('/proc/self/mountinfo', deadline=deadline).splitlines()
|
|
# Namespace-relative/subtree mounts are deliberately unsupported: their
|
|
# root cannot authenticate a host container's complete cgroup membership.
|
|
matches = [line.split(' - ', 1) for line in mounts if ' - ' in line]
|
|
if not any(left.split()[3:5] == ['/', CGROUP_ROOT] and right.split()[0] == 'cgroup2'
|
|
for left, right in matches):
|
|
raise CgroupUnavailable('unsupported-or-unavailable-host-cgroup2-mount')
|
|
membership = read_bounded('/proc/' + str(pid) + '/cgroup', deadline=deadline)
|
|
lines = membership.splitlines()
|
|
if len(lines) != 1 or not lines[0].startswith('0::/'):
|
|
raise CgroupUnavailable('unsupported-v1-hybrid-or-malformed-membership')
|
|
relative = lines[0][3:]
|
|
parts = relative.split('/')[1:]
|
|
if (not parts or any(not p or p in ('.', '..') or not re.fullmatch(r'[A-Za-z0-9_.:-]{1,128}', p) for p in parts)
|
|
or parts[-1] not in (identifier, 'docker-' + identifier + '.scope')):
|
|
raise ValueError('Unbound cgroup membership')
|
|
root = os.path.realpath(CGROUP_ROOT)
|
|
path = os.path.realpath(CGROUP_ROOT + relative)
|
|
if root != CGROUP_ROOT or path != root + relative or not path.startswith(root + os.sep):
|
|
raise ValueError('Cgroup path escaped')
|
|
return path, membership
|
|
|
|
|
|
def process_generation(pid, deadline=None):
|
|
check_deadline(deadline)
|
|
raw = read_bounded('/proc/' + str(pid) + '/stat', deadline=deadline)
|
|
prefix, suffix = raw.rsplit(')', 1)
|
|
if prefix.split(' ', 1)[0] != str(pid) or ' (' not in prefix:
|
|
raise ValueError('Invalid process identity')
|
|
fields = suffix.split()
|
|
if len(fields) < 20:
|
|
raise ValueError('Incomplete process identity')
|
|
start = parse_counter(fields[19])
|
|
root = os.stat('/proc/' + str(pid) + '/root')
|
|
return start, root.st_dev, root.st_ino
|
|
|
|
|
|
def unavailable_cgroup(reason):
|
|
return {'version': 2, 'status': 'unavailable', 'reason': reason,
|
|
'metrics': {name: None for name in CGROUP_FILES}, 'unavailable': {name: reason for name in CGROUP_FILES}}
|
|
|
|
|
|
def collect_container(identifier, deadline=None):
|
|
check_deadline(deadline)
|
|
before = docker_get('/containers/' + identifier + '/json', deadline)
|
|
stats = docker_get('/containers/' + identifier + '/stats?stream=false&one-shot=true', deadline)
|
|
check_deadline(deadline)
|
|
result = project_container(before, stats, identifier)
|
|
cgroup = unavailable_cgroup('identity-unavailable')
|
|
try:
|
|
identity = container_identity(before, identifier)
|
|
pid = identity[3]
|
|
generation = process_generation(pid, deadline)
|
|
path, membership = cgroup_location(pid, identifier, deadline)
|
|
directory = os.stat(path)
|
|
if directory.st_dev != os.stat(CGROUP_ROOT).st_dev:
|
|
raise ValueError('Cgroup filesystem changed')
|
|
metrics, missing = {}, {}
|
|
for name in CGROUP_FILES:
|
|
check_deadline(deadline)
|
|
try:
|
|
candidate = path + '/' + name
|
|
if os.path.realpath(candidate) != candidate:
|
|
raise ValueError('Symlinked metric')
|
|
raw = read_bounded(candidate, directory.st_dev, deadline=deadline)
|
|
check_deadline(deadline)
|
|
if name == 'memory.events':
|
|
metrics[name] = parse_events(raw, ('low', 'high', 'max', 'oom', 'oom_kill'))
|
|
elif name == 'memory.swap.events':
|
|
metrics[name] = parse_events(raw, ('high', 'max', 'fail'))
|
|
elif name in ('memory.max', 'memory.swap.max'):
|
|
metrics[name] = parse_limit(raw)
|
|
elif name == 'cpu.stat':
|
|
metrics[name] = parse_cpu_throttling(raw)
|
|
else:
|
|
metrics[name] = parse_counter(raw)
|
|
except (OSError, ValueError, UnicodeError):
|
|
metrics[name] = None
|
|
missing[name] = 'missing-unreadable-or-malformed'
|
|
check_deadline(deadline)
|
|
after_path, after_membership = cgroup_location(pid, identifier, deadline)
|
|
after_directory = os.stat(path)
|
|
if (process_generation(pid, deadline) != generation or after_path != path or after_membership != membership
|
|
or (directory.st_dev, directory.st_ino) != (after_directory.st_dev, after_directory.st_ino)):
|
|
cgroup = unavailable_cgroup('process-or-cgroup-changed')
|
|
else:
|
|
cgroup = {'version': 2, 'status': 'partial' if missing else 'available',
|
|
'reason': 'some-metrics-unavailable' if missing else None,
|
|
'metrics': metrics, 'unavailable': missing,
|
|
'peakScope': 'cgroup lifetime or since external reset; not an isolated workload peak',
|
|
'provenance': 'fixed host cgroup2 mount + exact Docker ID membership + process/root generation'}
|
|
except CgroupUnavailable as error:
|
|
cgroup = unavailable_cgroup(str(error))
|
|
except (OSError, ValueError, KeyError, IndexError, UnicodeError):
|
|
cgroup = unavailable_cgroup('identity-process-or-cgroup-unavailable-or-unsupported')
|
|
# Even failed cgroup observations are bracketed by a fresh Docker inspect.
|
|
check_deadline(deadline)
|
|
after = docker_get('/containers/' + identifier + '/json', deadline)
|
|
check_deadline(deadline)
|
|
try:
|
|
stable = container_identity(before, identifier) == container_identity(after, identifier)
|
|
except (ValueError, KeyError, TypeError):
|
|
stable = False
|
|
if not stable:
|
|
cgroup = unavailable_cgroup('container-identity-changed-or-unavailable')
|
|
limits_stable = resource_limits_stable(before, after) if stable else None
|
|
if stable and limits_stable is not True and cgroup['status'] != 'unavailable':
|
|
for key in ('memory.max', 'memory.swap.max'):
|
|
cgroup['metrics'][key] = None
|
|
cgroup['unavailable'][key] = 'container-resource-limits-changed-or-unavailable'
|
|
cgroup['status'] = 'partial'
|
|
cgroup['reason'] = 'some-metrics-unavailable'
|
|
result.update({'resourceLimitsStableDuringSample': limits_stable,
|
|
'pid': before['State'].get('Pid'), 'identityStableDuringSample': stable,
|
|
'runningAfter': after.get('State', {}).get('Running'),
|
|
'oomKilledAfter': after.get('State', {}).get('OOMKilled'), 'cgroupV2': cgroup})
|
|
return result
|
|
|
|
|
|
def docker_get(path, deadline=None):
|
|
deadline = time.monotonic() + 5 if deadline is None else deadline
|
|
check_deadline(deadline)
|
|
connection = http.client.HTTPConnection('localhost', timeout=min(5, deadline - time.monotonic()))
|
|
owned_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
connection.sock = owned_socket
|
|
# A socket timeout alone permits an endless body that drips bytes. The
|
|
# watchdog owns precisely this request's socket, including after HTTP's
|
|
# Connection:close detaches it from HTTPConnection into HTTPResponse.
|
|
def expire():
|
|
try:
|
|
owned_socket.shutdown(socket.SHUT_RDWR)
|
|
except OSError:
|
|
pass
|
|
finally:
|
|
owned_socket.close()
|
|
watchdog = threading.Timer(max(0, deadline - time.monotonic()), expire)
|
|
watchdog.daemon = True
|
|
watchdog_started = False
|
|
response = None
|
|
try:
|
|
watchdog.start()
|
|
watchdog_started = True
|
|
owned_socket.settimeout(min(5, max(0.001, deadline - time.monotonic())))
|
|
check_deadline(deadline)
|
|
owned_socket.connect('/var/run/docker.sock')
|
|
check_deadline(deadline)
|
|
connection.request('GET', '/v1.45' + path, headers={'Connection': 'close'})
|
|
response = connection.getresponse()
|
|
raw = response.read(MAX_RESPONSE + 1)
|
|
check_deadline(deadline)
|
|
if response.status != 200 or len(raw) > MAX_RESPONSE:
|
|
raise ValueError('Docker observation unavailable')
|
|
result = json.loads(raw)
|
|
check_deadline(deadline)
|
|
return result
|
|
except Exception:
|
|
check_deadline(deadline)
|
|
raise
|
|
finally:
|
|
watchdog.cancel()
|
|
if watchdog_started:
|
|
watchdog.join()
|
|
try:
|
|
if response is not None:
|
|
response.close()
|
|
finally:
|
|
try:
|
|
connection.close()
|
|
finally:
|
|
owned_socket.close()
|
|
|
|
|
|
def nonnegative(value):
|
|
if type(value) is not int or value < 0:
|
|
raise ValueError('Invalid numeric observation')
|
|
return value
|
|
|
|
|
|
def parse_host(meminfo, vmstat):
|
|
memory = {}
|
|
for line in meminfo.splitlines():
|
|
fields = line.split()
|
|
if fields and fields[0] in ('MemTotal:', 'MemAvailable:', 'SwapTotal:', 'SwapFree:'):
|
|
if len(fields) != 3 or fields[2] != 'kB':
|
|
raise ValueError('Invalid meminfo units')
|
|
memory[fields[0][:-1] + 'Bytes'] = nonnegative(int(fields[1])) * 1024
|
|
if len(memory) != 4 or memory['MemAvailableBytes'] > memory['MemTotalBytes'] or memory['SwapFreeBytes'] > memory['SwapTotalBytes']:
|
|
raise ValueError('Incomplete host memory observation')
|
|
for line in vmstat.splitlines():
|
|
fields = line.split()
|
|
if fields and fields[0] in ('pswpin', 'pswpout', 'oom_kill'):
|
|
memory[fields[0]] = nonnegative(int(fields[1]))
|
|
return memory
|
|
|
|
|
|
def project_container(inspect, stats, identifier):
|
|
if inspect.get('Id') != identifier or stats.get('id') != identifier:
|
|
raise ValueError('Pinned container changed')
|
|
state = inspect['State']
|
|
memory = stats['memory_stats']
|
|
usage = nonnegative(memory['usage'])
|
|
inactive = memory.get('stats', {}).get('inactive_file')
|
|
if inactive is not None:
|
|
inactive = nonnegative(inactive)
|
|
if inactive > usage:
|
|
raise ValueError('Invalid cache observation')
|
|
return {
|
|
'id': identifier, 'imageId': inspect['Image'],
|
|
'startedAt': state['StartedAt'], 'running': state['Running'] is True,
|
|
'oomKilled': state.get('OOMKilled'), 'restartCount': inspect.get('RestartCount'),
|
|
'usageIncludingCacheBytes': usage, 'inactiveFileBytes': inactive,
|
|
'usageMinusInactiveFileBytes': usage - inactive if inactive is not None else None,
|
|
'configuredLimitBytes': nonnegative(inspect['HostConfig']['Memory']),
|
|
'configuredMemorySwapBytes': configured_memory_swap(inspect),
|
|
'configuredMemorySwapStatus': 'available' if configured_memory_swap(inspect) is not None else 'unavailable',
|
|
'reportedLimitBytes': nonnegative(memory['limit']),
|
|
'swapUsageBytes': memory.get('swap_usage'),
|
|
}
|
|
|
|
|
|
def summarize(samples):
|
|
if not samples:
|
|
raise ValueError('No observations')
|
|
baseline = {c['id']: (c['imageId'], c['startedAt'], c['restartCount'], c.get('pid'), c.get('oomKilled')) for c in samples[0]['containers']}
|
|
stable = all({c['id']: (c['imageId'], c['startedAt'], c['restartCount'], c.get('pid'), c.get('oomKilled')) for c in sample['containers']} == baseline
|
|
and all(c.get('identityStableDuringSample', True) for c in sample['containers']) for sample in samples)
|
|
counters = {}
|
|
for field in ('pswpin', 'pswpout', 'oom_kill'):
|
|
first = samples[0]['host'].get(field)
|
|
last = samples[-1]['host'].get(field)
|
|
counters[field + 'Delta'] = last - first if first is not None and last is not None and last >= first else None
|
|
return {
|
|
'samples': len(samples), 'elapsedMs': samples[-1]['elapsedMs'],
|
|
'hostMemTotalBytes': samples[0]['host']['MemTotalBytes'],
|
|
'minHostAvailableBytes': min(s['host']['MemAvailableBytes'] for s in samples),
|
|
'maxObservedContainerUsageIncludingCacheBytes': max(sum(c['usageIncludingCacheBytes'] for c in s['containers']) for s in samples),
|
|
'containerIdentityAndStartStable': stable,
|
|
'allObservedRunning': all(c['running'] is True and c.get('runningAfter') is True
|
|
and c.get('identityStableDuringSample') is True for s in samples for c in s['containers']),
|
|
'anyObservedOomKilled': any(c['oomKilled'] is True or c.get('oomKilledAfter') is True for s in samples for c in s['containers']),
|
|
'allCgroupV2MetricsAvailable': all(s['containers'] and all(c.get('cgroupV2', {}).get('status') == 'available' for c in s['containers']) for s in samples),
|
|
**counters,
|
|
'allResourceLimitsStableDuringSample': all(c.get('resourceLimitsStableDuringSample') is True for s in samples for c in s['containers']) and all(s['containers'] for s in samples),
|
|
'allHostMemoryPressureAvailable': all(s['host'].get('memoryPressure', {}).get('status') == 'available' for s in samples),
|
|
'acceptancePassed': False,
|
|
'scope': 'sampled evidence only; correlate with workload, limits, errors and latency; unsampled peaks are unknown',
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--daemon-id', required=True)
|
|
parser.add_argument('--container', action='append', required=True)
|
|
parser.add_argument('--samples', type=int, default=30)
|
|
parser.add_argument('--interval', type=float, default=1)
|
|
parser.add_argument('--output', required=True)
|
|
args = parser.parse_args()
|
|
if os.name != 'posix' or not hasattr(socket, 'AF_UNIX'):
|
|
parser.error('Run on the Linux Docker host')
|
|
if not re.fullmatch(r'[A-Za-z0-9-]{8,128}', args.daemon_id) or not 1 <= len(args.container) <= 16 or len(set(args.container)) != len(args.container) or any(not re.fullmatch(r'[a-f0-9]{64}', c) for c in args.container):
|
|
parser.error('Pin exact daemon ID and 1..16 distinct full container IDs')
|
|
if not 1 <= args.samples <= 300 or not 0.2 <= args.interval <= 10 or args.samples * args.interval > 300:
|
|
parser.error('Observation bound: 300 samples / 300 seconds')
|
|
started = time.monotonic()
|
|
deadline = started + 360
|
|
if docker_get('/info', deadline).get('ID') != args.daemon_id:
|
|
raise ValueError('Pinned Docker daemon changed')
|
|
samples = []
|
|
for index in range(args.samples):
|
|
check_deadline(deadline)
|
|
host = collect_host(deadline)
|
|
containers = []
|
|
for identifier in args.container:
|
|
check_deadline(deadline)
|
|
containers.append(collect_container(identifier, deadline))
|
|
check_deadline(deadline)
|
|
samples.append({'elapsedMs': round((time.monotonic() - started) * 1000), 'host': host, 'containers': containers})
|
|
if index + 1 < args.samples:
|
|
time.sleep(min(args.interval, max(0, deadline - time.monotonic())))
|
|
check_deadline(deadline)
|
|
if docker_get('/info', deadline).get('ID') != args.daemon_id:
|
|
raise ValueError('Pinned Docker daemon changed')
|
|
result = {'schemaVersion': 3, 'readOnlyDockerAndProc': True, 'serviceChanges': 0, 'summary': summarize(samples), 'observations': samples}
|
|
encoded = json.dumps(result, indent=2) + '\n'
|
|
check_deadline(deadline)
|
|
# Refuse to replace an existing evidence file. Never dump Docker env/logs.
|
|
with open(args.output, 'x', encoding='utf-8') as handle:
|
|
check_deadline(deadline)
|
|
handle.write(encoded)
|
|
check_deadline(deadline)
|
|
print(json.dumps(result['summary']))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except Exception as error:
|
|
print(json.dumps({'status': 'observation-refused', 'errorClass': type(error).__name__, 'serviceChanges': 0}))
|
|
raise SystemExit(1)
|