Skip to content

GitLab

F
fuss-manager
Commit 8517d383 authored by Elena Grandi's avatar Elena Grandi
Browse files
Options

flake8

parent 5188a187
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 317 additions and 327 deletions
manager/stores.py
View file @ 8517d383
......@@ -22,19 +22,30 @@ T = TypeVar("T")
# Convenience variable with all of the fields of a machine that can be
# saved as json
ALL_MACHINE_FIELDS = (
"mac", "ip", "name",
"first_seen", "last_seen", "registered",
"groups", "host_vars",
"facts", "facts_log",
)
"mac",
"ip",
"name",
"first_seen",
"last_seen",
"registered",
"groups",
"host_vars",
"facts",
"facts_log",
)
# Fields of a machine that we want to save in the stats file (excluding
# what is already saved in the inventory and what isn't sensible to
# save).
STATS_MACHINE_FIELDS = (
"mac", "ip", "name",
"first_seen", "last_seen", "registered",
"facts", "facts_log",
)
"mac",
"ip",
"name",
"first_seen",
"last_seen",
"registered",
"facts",
"facts_log",
)
class RRList(list):
......@@ -42,13 +53,14 @@ class RRList(list):
A Round Robin list of timestamped entries that will drop older
entries.
"""
def __init__(
self,
*args,
max_age: Optional[int] = 60, # days
max_size: Optional[int] = None,
**kw
):
self,
*args,
max_age: Optional[int] = 60, # days
max_size: Optional[int] = None,
**kw
):
super().__init__(*args, **kw)
self.max_age = max_age
self.max_size = max_size
......@@ -71,19 +83,20 @@ class Machine:
"""
Known data about a machine
"""
def __init__(
self,
mac: str,
ip: Optional[str] = None,
name: Optional[str] = None,
registered: Optional[bool] = None,
first_seen: Optional[float] = None,
last_seen: Optional[float] = None,
groups: Iterable[str] = None,
host_vars: Optional[dict] = None,
facts: Optional[dict] = None,
facts_log: Optional[list] = None,
):
self,
mac: str,
ip: Optional[str] = None,
name: Optional[str] = None,
registered: Optional[bool] = None,
first_seen: Optional[float] = None,
last_seen: Optional[float] = None,
groups: Iterable[str] = None,
host_vars: Optional[dict] = None,
facts: Optional[dict] = None,
facts_log: Optional[list] = None,
):
self.mac = mac
self.ip = ip
# A machine always needs a name
......@@ -161,10 +174,8 @@ class MachineStore:
"""
def __init__(
self,
event_hub,
config,
):
self, event_hub, config,
):
self.log = logging.getLogger(self.__class__.__name__)
self.event_hub = event_hub
# mac -> Machine
......@@ -178,10 +189,8 @@ class MachineStore:
self.resolver = aiodns.DNSResolver(loop=loop)
def get_machine(
self,
ip: Optional[str] = None,
name: Optional[str] = None,
):
self, ip: Optional[str] = None, name: Optional[str] = None,
):
"""
Get a machine by ip or name.
......@@ -209,12 +218,12 @@ class MachineStore:
old_groups = tuple(machine.groups)
machine.groups.append(group)
await self.event_hub.publish(
events.HostChangedEvent(
self,
mac=mac,
changes={
"groups": (old_groups, tuple(machine.groups)),
}))
events.HostChangedEvent(
self,
mac=mac,
changes={"groups": (old_groups, tuple(machine.groups))},
)
)
await self.flush()
async def remove_machine_from_group(self, mac: str, group: str):
......@@ -226,12 +235,12 @@ class MachineStore:
old_groups = tuple(machine.groups)
machine.groups.remove(group)
await self.event_hub.publish(
events.HostChangedEvent(
self,
mac=mac,
changes={
"groups": (old_groups, tuple(machine.groups)),
}))
events.HostChangedEvent(
self,
mac=mac,
changes={"groups": (old_groups, tuple(machine.groups))},
)
)
await self.flush()
async def host_seen(self, evt):
......@@ -268,14 +277,14 @@ class MachineStore:
machine = Machine.from_host_seen_event(evt)
self.machines[machine.mac] = machine
await self.event_hub.publish(
events.HostNewEvent(
self,
timestamp=evt.timestamp,
mac=machine.mac,
ip=machine.ip,
name=machine.name
)
)
events.HostNewEvent(
self,
timestamp=evt.timestamp,
mac=machine.mac,
ip=machine.ip,
name=machine.name,
)
)
await self.flush()
else:
# Existing machine: only keep track of data that changed
......@@ -286,66 +295,57 @@ class MachineStore:
self,
timestamp=evt.timestamp,
mac=evt.mac,
changes=changes
)
changes=changes,
)
)
await self.flush()
async def facts_loaded(self, evt):
machine = self.get_machine(name=evt.name)
old_facts_log = machine.facts_log.copy()
machine.facts_log.append({
'timestamp': evt.timestamp,
'result': 'SUCCESS',
'details': {},
})
machine.facts_log.append(
{'timestamp': evt.timestamp, 'result': 'SUCCESS', 'details': {}}
)
changes = {
"facts": (machine.facts, evt.facts),
"facts_log": (old_facts_log, machine.facts_log),
}
}
machine.facts = evt.facts
if not machine.registered:
machine.registered = True
changes['registered'] = (False, True)
await self.event_hub.publish(
events.HostChangedEvent(
self,
mac=machine.mac,
changes=changes
)
)
events.HostChangedEvent(self, mac=machine.mac, changes=changes)
)
async def facts_failed(self, evt):
machine = self.get_machine(name=evt.name)
old_facts_log = machine.facts_log.copy()
machine.facts_log.append({
'timestamp': time.time(),
'result': evt.result,
'details': evt.details,
})
machine.facts_log.append(
{
'timestamp': time.time(),
'result': evt.result,
'details': evt.details,
}
)
changes = {
"facts_log": (old_facts_log, machine.facts_log),
}
}
await self.event_hub.publish(
events.HostChangedEvent(
self,
mac=machine.mac,
changes=changes
)
)
events.HostChangedEvent(self, mac=machine.mac, changes=changes)
)
async def mac_to_ip(self, mac):
"""
Find the IP of a machine with a known mac address.
"""
proc = await asyncio.create_subprocess_shell(
'/usr/sbin/arp -n',
stdout=asyncio.subprocess.PIPE,
)
'/usr/sbin/arp -n', stdout=asyncio.subprocess.PIPE,
)
out = await proc.communicate()
for l in out[0].decode().split('\n'):
if l and mac in l:
return l.split()[0]
for line in out[0].decode().split('\n'):
if line and mac in line:
return line.split()[0]
return None
async def ip_to_mac(self, ip):
......@@ -355,7 +355,7 @@ class MachineStore:
proc = await asyncio.create_subprocess_shell(
'/usr/sbin/arp -n {ip}'.format(ip=ip),
stdout=asyncio.subprocess.PIPE,
)
)
out = await proc.communicate()
if ip in out[0].decode().split('\n')[1]:
return out[0].decode().split('\n')[1][33:50]
......@@ -396,7 +396,7 @@ class MachineStore:
'ssh -oStrictHostKeyChecking=yes root@{n} "whoami"'.format(n=name),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
)
out, err = await proc.communicate()
if b'No route to host' in err:
......@@ -434,20 +434,14 @@ class MachineStore:
ip = None
if not mac:
self.log.warning(
"Could not find a valid mac for {m}, it will be removed".format(
m=m[0],
)
)
"Could not find a valid mac for {m}, it will be removed".format(m=m[0],) # noqa: E501
)
continue
# and then we add the machine if it's not already available,
# and add it to the relevant groups and load its variables
if mac not in self.machines:
self.machines[mac] = Machine(
mac=mac,
name=name,
ip=ip,
)
self.machines[mac] = Machine(mac=mac, name=name, ip=ip,)
for var, value in m[1].items():
if var != 'macaddress':
self.machines[mac].host_vars[var] = value
......@@ -485,7 +479,7 @@ class MachineStore:
stats = {
m.mac: m.to_jsonable(fields=STATS_MACHINE_FIELDS)
for m in self.machines.values()
}
}
with open(self.stats_fn, 'w') as fp:
json.dump(stats, fp)
......@@ -494,6 +488,7 @@ class MockMachineStore(MachineStore):
"""
A machine store that does not require external resources.
"""
def __init__(self, *args, **kw):
super().__init__(*args, **kw)
with open('tests/data/local_network_data.json') as fp:
......@@ -549,12 +544,13 @@ class MockMachineStore(MachineStore):
return False
class SimpleAnsibleInventory():
class SimpleAnsibleInventory:
"""
Manage an ansible inventory file.
Only static formats are supported.
"""
# This is kept separated from the MachineStore to make it easier to
# reimplement it using ansible's own inventory parsing code.
# At the moment this is problematic because ansible in jessie is
......@@ -568,8 +564,8 @@ class SimpleAnsibleInventory():
'all': {
'hosts': {},
'children': {},
}
})
}
})
def __init__(self, fname: str, inv_format: Optional[str] = 'yaml'):
self.inventory = self._get_empty_inventory()
......@@ -578,10 +574,8 @@ class SimpleAnsibleInventory():
self.inv_format = inv_format
def load(
self,
fname: Optional[str] = None,
inv_format: Optional[str] = None
):
self, fname: Optional[str] = None, inv_format: Optional[str] = None
):
"""
Load inventory data.
......@@ -600,15 +594,13 @@ class SimpleAnsibleInventory():
raise ValueError(
'{} is not a supported format for an inventory'.format(
inv_format
)
)
)
self._load_vars()
def save(
self,
fname: Optional[str] = None,
inv_format: Optional[str] = None
):
self, fname: Optional[str] = None, inv_format: Optional[str] = None
):
"""
Save inventory data to file.
......@@ -627,8 +619,8 @@ class SimpleAnsibleInventory():
raise ValueError(
'{} is not a supported format for an inventory'.format(
inv_format
)
)
)
self._save_vars()
def _parse_ini(self, fname: str):
......@@ -637,17 +629,19 @@ class SimpleAnsibleInventory():
cur_group = inventory['all']['hosts']
try:
with open(fname) as fp:
for l in fp.readlines():
if not l.strip():
for line in fp.readlines():
if not line.strip():
continue
elif l.startswith('['):
group = l.strip('[]\n\r\t ')
elif line.startswith('['):
group = line.strip('[]\n\r\t ')
inventory['all']['children'][group] = {
'hosts': {},
}
cur_group = inventory['all']['children'][group]['hosts']
}
cur_group = inventory['all']['children'][group][
'hosts'
]
else:
split_l = l.strip().split()
split_l = line.strip().split()
cur_group[split_l[0]] = {}
for item in split_l[1:]:
try:
......@@ -673,19 +667,17 @@ class SimpleAnsibleInventory():
def _save_ini(self, fname: str):
with open(fname, 'w') as fp:
for h, v in self.inventory['all']['hosts'].items():
fp.write('{h} macaddress={m}\n'.format(
h=h,
m=v['macaddress']),
)
fp.write(
'{h} macaddress={m}\n'.format(h=h, m=v['macaddress']),
)
for c in self.inventory['all']['children']:
fp.write("[{}]\n".format(c))
for h, v in self.inventory[
'all'
]['children'][c]['hosts'].items():
fp.write('{h} macaddress={m}\n'.format(
h=h,
m=v['macaddress']),
)
for h, v in self.inventory['all']['children'][c][
'hosts'
].items():
fp.write(
'{h} macaddress={m}\n'.format(h=h, m=v['macaddress']),
)
def _save_yaml(self, fname: str):
yaml = ruamel.yaml.YAML()
......@@ -701,7 +693,9 @@ class SimpleAnsibleInventory():
for h, v in data['all']['children'][c]['hosts'].items():
mac = v.get('macaddress')
if mac:
data['all']['children'][c]['hosts'][h] = {'macaddress': mac}
data['all']['children'][c]['hosts'][h] = {
'macaddress': mac
}
else:
data['all']['children'][c]['hosts'][h] = {}
with open(fname, 'w') as fp:
......@@ -714,11 +708,12 @@ class SimpleAnsibleInventory():
if 'children' in self.inventory['all']:
for c in self.inventory['all']['children']:
for h, v in self.inventory['all']['children'][c][
'hosts'].items():
'hosts'
].items():
f(h, v)
def _load_host_vars(self, h, v):
var_file = os.path.join(self.var_dir, 'host_vars', h+'.yaml')
var_file = os.path.join(self.var_dir, 'host_vars', h + '.yaml')
if os.path.isfile(var_file):
yaml = ruamel.yaml.YAML()
with open(var_file) as fp:
......@@ -739,11 +734,9 @@ class SimpleAnsibleInventory():
yaml = ruamel.yaml.YAML()
yaml.explicit_start = True
os.makedirs(os.path.join(self.var_dir, 'host_vars'), exist_ok=True)
with open(os.path.join(
self.var_dir,
'host_vars',
host+'.yaml'
), 'w') as fp:
with open(
os.path.join(self.var_dir, 'host_vars', host + '.yaml'), 'w'
) as fp:
yaml.dump(data, fp)
def _save_vars(self):
......@@ -757,28 +750,29 @@ class SimpleAnsibleInventory():
if g == 'all':
self.inventory['all']['hosts'][machine.name] = {
'macaddress': machine.mac,
}
}
self.inventory['all']['hosts'][machine.name].update(
machine.host_vars)
machine.host_vars
)
else:
if g not in self.inventory['all']['children']:
self.inventory['all']['children'][g] = {
'hosts': {}
}
self.inventory['all']['children'][g] = {'hosts': {}}
self.inventory['all']['children'][g]['hosts'][
machine.name] = {
machine.name
] = {
'macaddress': machine.mac,
}
}
self.inventory['all']['children'][g]['hosts'][
machine.name].update(machine.host_vars)
machine.name
].update(machine.host_vars)
else:
self.inventory['all']['hosts'][machine.name] = {
'macaddress': machine.mac,
}
}
self.inventory['all']['hosts'][machine.name].update(
machine.host_vars)
machine.host_vars
)
def _group_machines(self, group, gname):
for m in group['hosts'].items():
......
tests/test_store.py
View file @ 8517d383
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment