Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
FUSS
fuss-manager
Commits
8517d383
Commit
8517d383
authored
Aug 12, 2020
by
Elena Grandi
Browse files
flake8
parent
5188a187
Changes
2
Expand all
Hide whitespace changes
Inline
Side-by-side
manager/stores.py
View file @
8517d383
...
...
@@ -22,19 +22,30 @@ T = TypeVar("T")
# Convenience variable with all of the fields of a machine that can be
# saved as json
ALL_MACHINE_FIELDS
=
(
"mac"
,
"ip"
,
"name"
,
"first_seen"
,
"last_seen"
,
"registered"
,
"groups"
,
"host_vars"
,
"facts"
,
"facts_log"
,
)
"mac"
,
"ip"
,
"name"
,
"first_seen"
,
"last_seen"
,
"registered"
,
"groups"
,
"host_vars"
,
"facts"
,
"facts_log"
,
)
# Fields of a machine that we want to save in the stats file (excluding
# what is already saved in the inventory and what isn't sensible to
# save).
STATS_MACHINE_FIELDS
=
(
"mac"
,
"ip"
,
"name"
,
"first_seen"
,
"last_seen"
,
"registered"
,
"facts"
,
"facts_log"
,
)
"mac"
,
"ip"
,
"name"
,
"first_seen"
,
"last_seen"
,
"registered"
,
"facts"
,
"facts_log"
,
)
class
RRList
(
list
):
...
...
@@ -42,13 +53,14 @@ class RRList(list):
A Round Robin list of timestamped entries that will drop older
entries.
"""
def
__init__
(
self
,
*
args
,
max_age
:
Optional
[
int
]
=
60
,
# days
max_size
:
Optional
[
int
]
=
None
,
**
kw
):
self
,
*
args
,
max_age
:
Optional
[
int
]
=
60
,
# days
max_size
:
Optional
[
int
]
=
None
,
**
kw
):
super
().
__init__
(
*
args
,
**
kw
)
self
.
max_age
=
max_age
self
.
max_size
=
max_size
...
...
@@ -71,19 +83,20 @@ class Machine:
"""
Known data about a machine
"""
def
__init__
(
self
,
mac
:
str
,
ip
:
Optional
[
str
]
=
None
,
name
:
Optional
[
str
]
=
None
,
registered
:
Optional
[
bool
]
=
None
,
first_seen
:
Optional
[
float
]
=
None
,
last_seen
:
Optional
[
float
]
=
None
,
groups
:
Iterable
[
str
]
=
None
,
host_vars
:
Optional
[
dict
]
=
None
,
facts
:
Optional
[
dict
]
=
None
,
facts_log
:
Optional
[
list
]
=
None
,
):
self
,
mac
:
str
,
ip
:
Optional
[
str
]
=
None
,
name
:
Optional
[
str
]
=
None
,
registered
:
Optional
[
bool
]
=
None
,
first_seen
:
Optional
[
float
]
=
None
,
last_seen
:
Optional
[
float
]
=
None
,
groups
:
Iterable
[
str
]
=
None
,
host_vars
:
Optional
[
dict
]
=
None
,
facts
:
Optional
[
dict
]
=
None
,
facts_log
:
Optional
[
list
]
=
None
,
):
self
.
mac
=
mac
self
.
ip
=
ip
# A machine always needs a name
...
...
@@ -161,10 +174,8 @@ class MachineStore:
"""
def
__init__
(
self
,
event_hub
,
config
,
):
self
,
event_hub
,
config
,
):
self
.
log
=
logging
.
getLogger
(
self
.
__class__
.
__name__
)
self
.
event_hub
=
event_hub
# mac -> Machine
...
...
@@ -178,10 +189,8 @@ class MachineStore:
self
.
resolver
=
aiodns
.
DNSResolver
(
loop
=
loop
)
def
get_machine
(
self
,
ip
:
Optional
[
str
]
=
None
,
name
:
Optional
[
str
]
=
None
,
):
self
,
ip
:
Optional
[
str
]
=
None
,
name
:
Optional
[
str
]
=
None
,
):
"""
Get a machine by ip or name.
...
...
@@ -209,12 +218,12 @@ class MachineStore:
old_groups
=
tuple
(
machine
.
groups
)
machine
.
groups
.
append
(
group
)
await
self
.
event_hub
.
publish
(
events
.
HostChangedEvent
(
self
,
mac
=
mac
,
changes
=
{
"groups"
:
(
old_groups
,
tuple
(
machine
.
groups
)),
})
)
events
.
HostChangedEvent
(
self
,
mac
=
mac
,
changes
=
{
"groups"
:
(
old_groups
,
tuple
(
machine
.
groups
))},
)
)
await
self
.
flush
()
async
def
remove_machine_from_group
(
self
,
mac
:
str
,
group
:
str
):
...
...
@@ -226,12 +235,12 @@ class MachineStore:
old_groups
=
tuple
(
machine
.
groups
)
machine
.
groups
.
remove
(
group
)
await
self
.
event_hub
.
publish
(
events
.
HostChangedEvent
(
self
,
mac
=
mac
,
changes
=
{
"groups"
:
(
old_groups
,
tuple
(
machine
.
groups
)),
})
)
events
.
HostChangedEvent
(
self
,
mac
=
mac
,
changes
=
{
"groups"
:
(
old_groups
,
tuple
(
machine
.
groups
))},
)
)
await
self
.
flush
()
async
def
host_seen
(
self
,
evt
):
...
...
@@ -268,14 +277,14 @@ class MachineStore:
machine
=
Machine
.
from_host_seen_event
(
evt
)
self
.
machines
[
machine
.
mac
]
=
machine
await
self
.
event_hub
.
publish
(
events
.
HostNewEvent
(
self
,
timestamp
=
evt
.
timestamp
,
mac
=
machine
.
mac
,
ip
=
machine
.
ip
,
name
=
machine
.
name
)
)
events
.
HostNewEvent
(
self
,
timestamp
=
evt
.
timestamp
,
mac
=
machine
.
mac
,
ip
=
machine
.
ip
,
name
=
machine
.
name
,
)
)
await
self
.
flush
()
else
:
# Existing machine: only keep track of data that changed
...
...
@@ -286,66 +295,57 @@ class MachineStore:
self
,
timestamp
=
evt
.
timestamp
,
mac
=
evt
.
mac
,
changes
=
changes
)
changes
=
changes
,
)
)
await
self
.
flush
()
async
def
facts_loaded
(
self
,
evt
):
machine
=
self
.
get_machine
(
name
=
evt
.
name
)
old_facts_log
=
machine
.
facts_log
.
copy
()
machine
.
facts_log
.
append
({
'timestamp'
:
evt
.
timestamp
,
'result'
:
'SUCCESS'
,
'details'
:
{},
})
machine
.
facts_log
.
append
(
{
'timestamp'
:
evt
.
timestamp
,
'result'
:
'SUCCESS'
,
'details'
:
{}}
)
changes
=
{
"facts"
:
(
machine
.
facts
,
evt
.
facts
),
"facts_log"
:
(
old_facts_log
,
machine
.
facts_log
),
}
}
machine
.
facts
=
evt
.
facts
if
not
machine
.
registered
:
machine
.
registered
=
True
changes
[
'registered'
]
=
(
False
,
True
)
await
self
.
event_hub
.
publish
(
events
.
HostChangedEvent
(
self
,
mac
=
machine
.
mac
,
changes
=
changes
)
)
events
.
HostChangedEvent
(
self
,
mac
=
machine
.
mac
,
changes
=
changes
)
)
async
def
facts_failed
(
self
,
evt
):
machine
=
self
.
get_machine
(
name
=
evt
.
name
)
old_facts_log
=
machine
.
facts_log
.
copy
()
machine
.
facts_log
.
append
({
'timestamp'
:
time
.
time
(),
'result'
:
evt
.
result
,
'details'
:
evt
.
details
,
})
machine
.
facts_log
.
append
(
{
'timestamp'
:
time
.
time
(),
'result'
:
evt
.
result
,
'details'
:
evt
.
details
,
}
)
changes
=
{
"facts_log"
:
(
old_facts_log
,
machine
.
facts_log
),
}
}
await
self
.
event_hub
.
publish
(
events
.
HostChangedEvent
(
self
,
mac
=
machine
.
mac
,
changes
=
changes
)
)
events
.
HostChangedEvent
(
self
,
mac
=
machine
.
mac
,
changes
=
changes
)
)
async
def
mac_to_ip
(
self
,
mac
):
"""
Find the IP of a machine with a known mac address.
"""
proc
=
await
asyncio
.
create_subprocess_shell
(
'/usr/sbin/arp -n'
,
stdout
=
asyncio
.
subprocess
.
PIPE
,
)
'/usr/sbin/arp -n'
,
stdout
=
asyncio
.
subprocess
.
PIPE
,
)
out
=
await
proc
.
communicate
()
for
l
in
out
[
0
].
decode
().
split
(
'
\n
'
):
if
l
and
mac
in
l
:
return
l
.
split
()[
0
]
for
l
ine
in
out
[
0
].
decode
().
split
(
'
\n
'
):
if
l
ine
and
mac
in
l
ine
:
return
l
ine
.
split
()[
0
]
return
None
async
def
ip_to_mac
(
self
,
ip
):
...
...
@@ -355,7 +355,7 @@ class MachineStore:
proc
=
await
asyncio
.
create_subprocess_shell
(
'/usr/sbin/arp -n {ip}'
.
format
(
ip
=
ip
),
stdout
=
asyncio
.
subprocess
.
PIPE
,
)
)
out
=
await
proc
.
communicate
()
if
ip
in
out
[
0
].
decode
().
split
(
'
\n
'
)[
1
]:
return
out
[
0
].
decode
().
split
(
'
\n
'
)[
1
][
33
:
50
]
...
...
@@ -396,7 +396,7 @@ class MachineStore:
'ssh -oStrictHostKeyChecking=yes root@{n} "whoami"'
.
format
(
n
=
name
),
stdout
=
asyncio
.
subprocess
.
PIPE
,
stderr
=
asyncio
.
subprocess
.
PIPE
,
)
)
out
,
err
=
await
proc
.
communicate
()
if
b
'No route to host'
in
err
:
...
...
@@ -434,20 +434,14 @@ class MachineStore:
ip
=
None
if
not
mac
:
self
.
log
.
warning
(
"Could not find a valid mac for {m}, it will be removed"
.
format
(
m
=
m
[
0
],
)
)
"Could not find a valid mac for {m}, it will be removed"
.
format
(
m
=
m
[
0
],)
# noqa: E501
)
continue
# and then we add the machine if it's not already available,
# and add it to the relevant groups and load its variables
if
mac
not
in
self
.
machines
:
self
.
machines
[
mac
]
=
Machine
(
mac
=
mac
,
name
=
name
,
ip
=
ip
,
)
self
.
machines
[
mac
]
=
Machine
(
mac
=
mac
,
name
=
name
,
ip
=
ip
,)
for
var
,
value
in
m
[
1
].
items
():
if
var
!=
'macaddress'
:
self
.
machines
[
mac
].
host_vars
[
var
]
=
value
...
...
@@ -485,7 +479,7 @@ class MachineStore:
stats
=
{
m
.
mac
:
m
.
to_jsonable
(
fields
=
STATS_MACHINE_FIELDS
)
for
m
in
self
.
machines
.
values
()
}
}
with
open
(
self
.
stats_fn
,
'w'
)
as
fp
:
json
.
dump
(
stats
,
fp
)
...
...
@@ -494,6 +488,7 @@ class MockMachineStore(MachineStore):
"""
A machine store that does not require external resources.
"""
def
__init__
(
self
,
*
args
,
**
kw
):
super
().
__init__
(
*
args
,
**
kw
)
with
open
(
'tests/data/local_network_data.json'
)
as
fp
:
...
...
@@ -549,12 +544,13 @@ class MockMachineStore(MachineStore):
return
False
class
SimpleAnsibleInventory
()
:
class
SimpleAnsibleInventory
:
"""
Manage an ansible inventory file.
Only static formats are supported.
"""
# This is kept separated from the MachineStore to make it easier to
# reimplement it using ansible's own inventory parsing code.
# At the moment this is problematic because ansible in jessie is
...
...
@@ -568,8 +564,8 @@ class SimpleAnsibleInventory():
'all'
:
{
'hosts'
:
{},
'children'
:
{},
}
})
}
})
def
__init__
(
self
,
fname
:
str
,
inv_format
:
Optional
[
str
]
=
'yaml'
):
self
.
inventory
=
self
.
_get_empty_inventory
()
...
...
@@ -578,10 +574,8 @@ class SimpleAnsibleInventory():
self
.
inv_format
=
inv_format
def
load
(
self
,
fname
:
Optional
[
str
]
=
None
,
inv_format
:
Optional
[
str
]
=
None
):
self
,
fname
:
Optional
[
str
]
=
None
,
inv_format
:
Optional
[
str
]
=
None
):
"""
Load inventory data.
...
...
@@ -600,15 +594,13 @@ class SimpleAnsibleInventory():
raise
ValueError
(
'{} is not a supported format for an inventory'
.
format
(
inv_format
)
)
)
self
.
_load_vars
()
def
save
(
self
,
fname
:
Optional
[
str
]
=
None
,
inv_format
:
Optional
[
str
]
=
None
):
self
,
fname
:
Optional
[
str
]
=
None
,
inv_format
:
Optional
[
str
]
=
None
):
"""
Save inventory data to file.
...
...
@@ -627,8 +619,8 @@ class SimpleAnsibleInventory():
raise
ValueError
(
'{} is not a supported format for an inventory'
.
format
(
inv_format
)
)
)
self
.
_save_vars
()
def
_parse_ini
(
self
,
fname
:
str
):
...
...
@@ -637,17 +629,19 @@ class SimpleAnsibleInventory():
cur_group
=
inventory
[
'all'
][
'hosts'
]
try
:
with
open
(
fname
)
as
fp
:
for
l
in
fp
.
readlines
():
if
not
l
.
strip
():
for
l
ine
in
fp
.
readlines
():
if
not
l
ine
.
strip
():
continue
elif
l
.
startswith
(
'['
):
group
=
l
.
strip
(
'[]
\n\r\t
'
)
elif
l
ine
.
startswith
(
'['
):
group
=
l
ine
.
strip
(
'[]
\n\r\t
'
)
inventory
[
'all'
][
'children'
][
group
]
=
{
'hosts'
:
{},
}
cur_group
=
inventory
[
'all'
][
'children'
][
group
][
'hosts'
]
}
cur_group
=
inventory
[
'all'
][
'children'
][
group
][
'hosts'
]
else
:
split_l
=
l
.
strip
().
split
()
split_l
=
l
ine
.
strip
().
split
()
cur_group
[
split_l
[
0
]]
=
{}
for
item
in
split_l
[
1
:]:
try
:
...
...
@@ -673,19 +667,17 @@ class SimpleAnsibleInventory():
def
_save_ini
(
self
,
fname
:
str
):
with
open
(
fname
,
'w'
)
as
fp
:
for
h
,
v
in
self
.
inventory
[
'all'
][
'hosts'
].
items
():
fp
.
write
(
'{h} macaddress={m}
\n
'
.
format
(
h
=
h
,
m
=
v
[
'macaddress'
]),
)
fp
.
write
(
'{h} macaddress={m}
\n
'
.
format
(
h
=
h
,
m
=
v
[
'macaddress'
]),
)
for
c
in
self
.
inventory
[
'all'
][
'children'
]:
fp
.
write
(
"[{}]
\n
"
.
format
(
c
))
for
h
,
v
in
self
.
inventory
[
'all'
][
'children'
][
c
][
'hosts'
].
items
():
fp
.
write
(
'{h} macaddress={m}
\n
'
.
format
(
h
=
h
,
m
=
v
[
'macaddress'
]),
)
for
h
,
v
in
self
.
inventory
[
'all'
][
'children'
][
c
][
'hosts'
].
items
():
fp
.
write
(
'{h} macaddress={m}
\n
'
.
format
(
h
=
h
,
m
=
v
[
'macaddress'
]),
)
def
_save_yaml
(
self
,
fname
:
str
):
yaml
=
ruamel
.
yaml
.
YAML
()
...
...
@@ -701,7 +693,9 @@ class SimpleAnsibleInventory():
for
h
,
v
in
data
[
'all'
][
'children'
][
c
][
'hosts'
].
items
():
mac
=
v
.
get
(
'macaddress'
)
if
mac
:
data
[
'all'
][
'children'
][
c
][
'hosts'
][
h
]
=
{
'macaddress'
:
mac
}
data
[
'all'
][
'children'
][
c
][
'hosts'
][
h
]
=
{
'macaddress'
:
mac
}
else
:
data
[
'all'
][
'children'
][
c
][
'hosts'
][
h
]
=
{}
with
open
(
fname
,
'w'
)
as
fp
:
...
...
@@ -714,11 +708,12 @@ class SimpleAnsibleInventory():
if
'children'
in
self
.
inventory
[
'all'
]:
for
c
in
self
.
inventory
[
'all'
][
'children'
]:
for
h
,
v
in
self
.
inventory
[
'all'
][
'children'
][
c
][
'hosts'
].
items
():
'hosts'
].
items
():
f
(
h
,
v
)
def
_load_host_vars
(
self
,
h
,
v
):
var_file
=
os
.
path
.
join
(
self
.
var_dir
,
'host_vars'
,
h
+
'.yaml'
)
var_file
=
os
.
path
.
join
(
self
.
var_dir
,
'host_vars'
,
h
+
'.yaml'
)
if
os
.
path
.
isfile
(
var_file
):
yaml
=
ruamel
.
yaml
.
YAML
()
with
open
(
var_file
)
as
fp
:
...
...
@@ -739,11 +734,9 @@ class SimpleAnsibleInventory():
yaml
=
ruamel
.
yaml
.
YAML
()
yaml
.
explicit_start
=
True
os
.
makedirs
(
os
.
path
.
join
(
self
.
var_dir
,
'host_vars'
),
exist_ok
=
True
)
with
open
(
os
.
path
.
join
(
self
.
var_dir
,
'host_vars'
,
host
+
'.yaml'
),
'w'
)
as
fp
:
with
open
(
os
.
path
.
join
(
self
.
var_dir
,
'host_vars'
,
host
+
'.yaml'
),
'w'
)
as
fp
:
yaml
.
dump
(
data
,
fp
)
def
_save_vars
(
self
):
...
...
@@ -757,28 +750,29 @@ class SimpleAnsibleInventory():
if
g
==
'all'
:
self
.
inventory
[
'all'
][
'hosts'
][
machine
.
name
]
=
{
'macaddress'
:
machine
.
mac
,
}
}
self
.
inventory
[
'all'
][
'hosts'
][
machine
.
name
].
update
(
machine
.
host_vars
)
machine
.
host_vars
)
else
:
if
g
not
in
self
.
inventory
[
'all'
][
'children'
]:
self
.
inventory
[
'all'
][
'children'
][
g
]
=
{
'hosts'
:
{}
}
self
.
inventory
[
'all'
][
'children'
][
g
]
=
{
'hosts'
:
{}}
self
.
inventory
[
'all'
][
'children'
][
g
][
'hosts'
][
machine
.
name
]
=
{
machine
.
name
]
=
{
'macaddress'
:
machine
.
mac
,
}
}
self
.
inventory
[
'all'
][
'children'
][
g
][
'hosts'
][
machine
.
name
].
update
(
machine
.
host_vars
)
machine
.
name
].
update
(
machine
.
host_vars
)
else
:
self
.
inventory
[
'all'
][
'hosts'
][
machine
.
name
]
=
{
'macaddress'
:
machine
.
mac
,
}
}
self
.
inventory
[
'all'
][
'hosts'
][
machine
.
name
].
update
(
machine
.
host_vars
)
machine
.
host_vars
)
def
_group_machines
(
self
,
group
,
gname
):
for
m
in
group
[
'hosts'
].
items
():
...
...
tests/test_store.py
View file @
8517d383
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment