vmbuilder/agent
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

update

Browse source
This commit is contained in:
Vyacheslav Anzhiganov 2016-04-20 03:39:25 +03:00
parent 41f1d73362
commit 7c79b6c5da
15 changed files with 312 additions and 579 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,2 +1 @@
*.pyc
config.py

4
README.md
View file

@ -1,6 +1,8 @@
# SWSCloudNode
Version: 1.0
# Install
## Install
```
$ apt-get install sqlite3

123
node/__init__.py → SWSCloudNode/__init__.py
View file

@ -1,51 +1,85 @@
# coding: utf-8
import config
import os
import logging
import subprocess
import sys
import json
import shutil
import socket
# import dnsmasq
import lxc
# import nginx
import os
import subprocess
import commands
import requests
from SWSCloudNode.settings import settings
from SWSCloudNode.logger import logging
from SWSCloudNode import lxc
class NodeClient():
class Tasks:
def __init__(self):
self.endpoint = settings.get('server', 'endpoint')
self.id = settings.get('server', 'id')
self.secret = settings.get('server', 'secret')
def get_item(self):
try:
response = requests.get('%s/server_api/task' % self.endpoint, auth=(self.id, self.secret))
except Exception as e:
logging.error('no connection with %s' % self.endpoint)
return None
else:
if response.status_code == 200:
return response.json()
logging.error("Unexpected status code: %d" % response.status_code)
return None
class Node:
def tasks_get(self):
response = requests.get('http://%s/server_api/tasks?node_id=%s&node_secret=%s' % (config.server, config.node_id, config.node_secret))
return response.json()
def task_status_update(self, task_id, status):
response = requests.get('http://%s/server_api/task_status_update?node_id=%s&node_secret=%s&task_id=%s&status=%s' % (config.server, config.node_id, config.node_secret, task_id, status))
return response.json()
def report_container_status(self, container_id, statistics):
data = {
'node_id': config.node_id,
'node_secret': config.node_secret,
'status': json.dumps(statistics)
try:
response = requests.get(
'%s/server_api/tasks' % settings.get('server', 'endpoint'),
auth=(settings.get('server', 'id'), settings.get('server', 'secret'))
)
except Exception as e:
sys.exit('no connection with %s' % settings.get('server', 'endpoint'))
# print e
else:
return {
'status': response.status_code,
'results': response.json()
}
print statistics
# for i in statistics:
# data[i] = statistics[i]
def task_status_update(self, task_id, status):
response = requests.put(
'%s/server_api/tasks/%s' % (
settings.get('server', 'endpoint'),
task_id
),
auth=(
settings.get('server', 'id'),
settings.get('server', 'secret'),
),
data={
"status": status
}
)
return response.json()
def report_container_stats(self, container_id, statistics):
response = requests.post(
'http://%s/server_api/report/container_status' % config.server,
data=data
'%s/server_api/containers/stats/%s' % (
settings.get('server', 'endpoint'),
container_id
),
auth=(settings.get('server', 'id'), settings.get('server', 'secret')),
data={
'status': json.dumps(statistics)
}
)
if response.status_code == 200:
return response.json()
# print response.text
return False
def __container_config_create(container_id, link, ipv4, ipv6):
cfg = []
# TODO: подумать куда переместить
def container_config_create(self, container_id, link, ipv4, ipv6):
cfg = list()
cfg.append("lxc.network.type = veth")
cfg.append("lxc.network.flags = up")
cfg.append("lxc.network.name = eth0")
@ -63,13 +97,13 @@ def __container_config_create(container_id, link, ipv4, ipv6):
config_file = '/var/lib/gocloud/node/configs/%s.config' % container_id
cfgfile = open(config_file, 'w')
cfgfile.write('\n'.join(cfg))
cfgfile.write('\n')
cfgfile.close()
cfg_file = open(config_file, 'w')
cfg_file.write('\n'.join(cfg))
cfg_file.write('\n')
cfg_file.close()
return True
def __container_authkey_create(container_id, auth_key):
def container_authkey_create(self, container_id, auth_key):
# create ssh_key.pub
authkey_file = '/var/lib/gocloud/node/auth-keys/%s.pub' % container_id
ak = open(authkey_file, 'w')
@ -79,7 +113,7 @@ def __container_authkey_create(container_id, auth_key):
return True
class Task(NodeClient):
class Task(Node):
def interface2ip(self):
# intf = open(self.settings['proxy_interface'], 'r').read().split('\n')[0]
interface = "eth0"
@ -267,18 +301,3 @@ class Task(NodeClient):
print "structure version not supported"
return None
class Report():
# def __init__(self, auth):
def container_info(self, data):
"""
Send container info to server
:param data:
:return:
"""
response = TCPClient().request(Request().build("report_container_info", config.auth, data))
if response['status'] == 0:
return True
return False

0
auth-keys/README → SWSCloudNode/libvirt/__init__.py
View file

8
SWSCloudNode/logger.py Normal file
View file

@ -0,0 +1,8 @@
import logging
FORMAT = '%(asctime)-15s NODEAGENT %(levelname)s: %(message)s'
logging.basicConfig(format=FORMAT)
# logger = logging.getLogger('tcpserver')
# logger.warning('Protocol problem: %s' % 'connection reset')

16
SWSCloudNode/reports.py Normal file
View file

@ -0,0 +1,16 @@
from SWSCloudNode.settings import settings
class Report:
# def __init__(self, auth):
def container_info(self, data):
"""
Send container info to server
:param data:
:return:
"""
response = TCPClient().request(Request().build("report_container_info", config.auth, data))
if response['status'] == 0:
return True
return False

39
SWSCloudNode/settings.py Normal file
View file

@ -0,0 +1,39 @@
# coding: utf-8
import sys
import os
import ConfigParser
from .logger import logging
default_file = '/etc/sws/cloud/node.ini'
settings_file = os.getenv('CLOUD_SETTINGS_FILE', default_file)
# setting file read
settings = ConfigParser.ConfigParser()
if os.path.exists(settings_file):
settings.read(settings_file)
if not settings.has_section('server'):
logging.error("No section: 'server'")
sys.exit()
if not settings.has_section('node'):
logging.error("No section: 'node'")
sys.exit()
if not settings.has_option('node', 'interface'):
logging.error("No option 'interface' in section: 'node'")
sys.exit()
else:
if settings.get('node', 'interface') not in os.listdir('/sys/class/net/'):
logging.error('Interface not found: %s' % settings.get('node', 'interface'))
sys.exit()
if not settings.has_option('node', 'sleep'):
logging.error("No option 'sleep' in section: 'node'")
sys.exit()
if not settings.has_option('container', 'packages'):
logging.error("No option 'packages' in section: 'container'")
sys.exit()
else:
sys.exit('settings file not found: %s' % settings_file)

5
cloud_node.py Normal file
View file

@ -0,0 +1,5 @@
#!/usr/bin/env python
from SWSCloudNode.settings import settings
print settings.get('server', '')

70
cloud_node_agent.py
View file

@ -1,22 +1,40 @@
# coding: utf-8
import requests
import config
import ConfigParser
import lxc
import node
import time
from SWSCloudNode.settings import settings
from SWSCloudNode.logger import logging
from SWSCloudNode import Node
from SWSCloudNode import Tasks
from SWSCloudNode import lxc
nodeclient = node.NodeClient()
tasks = nodeclient.tasks_get()
allowed_actions = [
'container_delete',
'container_create',
'container_start',
'container_stop',
'container_restart'
]
logging.debug("Application started")
while True:
time.sleep(settings.getfloat('node', 'sleep'))
nodeclient = Node()
task_data = Tasks().get_item()
if task_data is not None and 'task' in task_data:
if task_data['task']['task'] not in allowed_actions:
logging.critical("Task not allowed: %s" % task_data['task']['task'])
continue
task = task_data['task']
for task in tasks['results']:
# container_create
print task
print '---------------------'
if task['task'] == 'container_create':
# TODO: take from task
template = 'ubuntu'
if task['task'] == 'container_create':
# TODO: update task status to 1
nodeclient.task_status_update(task['id'], 1)
@ -30,32 +48,40 @@ for task in tasks['results']:
ipv6['ipv6'] = task['plain']['ipv6']
ipv6['ipv6_gateway'] = task['plain']['ipv6_gateway']
node.__container_config_create(container_id, config.interface, ipv4, ipv6)
nodeclient.container_config_create(container_id, settings.get('node', 'interface'), ipv4, ipv6)
container_config_file = '/var/lib/gocloud/node/configs/%s.config' % container_id
# create ssh_key.pub
param_auth_key = ''
if 'ssh_key' in task['plain'] and task['plain']['ssh_key']:
node.__container_authkey_create(container_id, task['plain']['ssh_key'])
nodeclient.container_authkey_create(container_id, task['plain']['ssh_key'])
param_auth_key = '--auth-key /var/lib/gocloud/node/auth-keys/%s.pub' % container_id
param_packages = ''
if template == 'ubuntu':
param_packages = '--packages %s' % ','.join(config.packages['ubuntu'])
param_packages = '--packages %s' % ','.join(settings.get('node', 'packages'))
# create container
lxc.lxc().create(
container_id,
container_config_file,
'ubuntu',
None,
"%s --user %s --password %s %s" % (
name=container_id,
config_file=container_config_file,
template='ubuntu',
backing_store=None,
template_options=' '.join([
param_auth_key,
'--user',
task['plain']['username'],
'--password',
task['plain']['password'],
param_packages
param_packages]
)
# template_options=""" %s --user %s --password %s %s """ % (
# param_auth_key,
# task['plain']['username'],
# task['plain']['password'],
# param_packages
# )
)
lxc.lxc().start(container_id)
@ -75,7 +101,7 @@ for task in tasks['results']:
container_id = task['plain']['container_id']
lxc.lxc().stop(container_id)
lxc.lxc().start(container_id)
nodeclient.task_status_update(task['id'], 2)
nodeclient.task_status_update(task['task']['id'], 2)
# container_stop
if task['task'] == 'container_stop':
@ -92,6 +118,6 @@ for task in tasks['results']:
try:
lxc.lxc().destroy(container_id)
except Exception as e:
print e
logging.warning(e)
pass
nodeclient.task_status_update(task['id'], 2)

17
cloud_node_statistics.py
View file

@ -1,19 +1,16 @@
# coding: utf-8
import requests
import config
import ConfigParser
import lxc
import node
from SWSCloudNode import Node
from SWSCloudNode import lxc
clslxc = lxc.lxc()
clsnode = node.NodeClient()
cls_lxc = lxc.lxc()
cls_node = Node()
containers = clslxc.list()
containers = cls_lxc.list()
for container in containers:
# print container
info = clslxc.info(container)
info = cls_lxc.info(container)
info['container_id'] = info['name']
print clsnode.report_container_status(info['container_id'], info)
print cls_node.report_container_stats(info['container_id'], info)
# print info

7
config.py.origin
View file

@ -1,7 +0,0 @@
server = 'gocloud.ru'
node_id = ""
node_secret = ""
interface = 'br0:0'
packages = {
'ubuntu': ['fail2ban']
}

0
configs/README
View file

14
extra/node_settings.ini Normal file
View file

@ -0,0 +1,14 @@
[server]
endpoint = http://api.gocloud.ru/api
id = 123
secret = 123
[node]
interface = br0:0
sleep = 1
;storage ????
;storage = lvm|folder
;storage = /var/lib/lxc
[container]
packages = fail2ban, mc, openssh-server

407
lxc/__init__.py
View file

@ -1,407 +0,0 @@
# coding: utf-8
import subprocess
import logging
import threading
import select
import pty
import os
import signal
class ContainerAlreadyExists(Exception):
pass
class ContainerAlreadyRunning(Exception):
pass
class ContainerNotExists(Exception):
pass
_logger = logging.getLogger("pylxc")
_monitor = None
class _LXCMonitor(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self._process = None
self._monitors = {}
def run(self):
master, slave = pty.openpty()
cmd = ['/usr/bin/lxc-monitor', '-n', '.*']
self._process = subprocess.Popen(cmd, stdout=slave, bufsize=1)
stdout = os.fdopen(master)
while self._process.poll() is None:
ready, _, _ = select.select([stdout], [], [], 0.1)
if ready:
logging.debug("Waiting for state change")
state = stdout.readline()
inf = state.strip().split()
container = inf[0].strip("'")
state = inf[-1].strip('[]')
if container in self._monitors:
logging.debug("State of container '%s' changed to '%s'", container, state)
self._monitors[container](state)
_logger.info("LXC Monitor stopped!")
def add_monitor(self, name, callback):
self._monitors[name] = callback
def rm_monitor(self, name):
self._monitors.pop(name)
def is_monitored(self, name):
return name in self._monitors
def kill(self):
try:
self._process.terminate()
self._process.wait()
except:
pass
self.join()
class lxc():
def __init__(self):
logging.debug("")
def list(self, status=None):
"""
:return: ['container_first', 'container_second']
"""
if status in ['active', 'frozen', 'running', 'stopped', 'nesting']:
path = "--%s" % status
else:
path = ""
cmd = ['/usr/bin/lxc-ls', path]
out = subprocess.check_output(cmd).splitlines()
# print out
return out
def exists(self, name):
"""
checks if a given container is defined or not
"""
if name in self.list():
return True
return False
def start(self, name, config_file=None):
"""
starts a container in daemon mode
"""
if not self.exists(name):
raise ContainerNotExists("The container (%s) does not exist!" % name)
if name in self.list("running"):
raise ContainerAlreadyRunning('The container %s is already started!' % name)
cmd = ['lxc-start', '-n', name, '-d']
if config_file:
cmd += ['-f', config_file]
return subprocess.check_call(cmd)
def stop(self, name):
"""
stops a container
"""
if not self.exists(name):
raise ContainerNotExists("The container (%s) does not exist!" % name)
cmd = ['/usr/bin/lxc-stop', '-n', name]
try:
result = subprocess.check_call(cmd)
return True
except Exception as e:
return False
def destroy(self, name):
"""
removes a container [stops a container if it's running and]
raises ContainerNotExists exception if the specified name is not created
"""
if not self.exists(name):
raise ContainerNotExists("The container (%s) does not exist!" % name)
# todo: check status. If status not STOPPED - run method self.stop(name)
# todo: add condition
self.stop(name)
cmd = ['/usr/bin/lxc-destroy', '-f', '-n', name]
return subprocess.check_call(cmd)
def info(self, name):
"""
returns info dict about the specified container
"""
#
if not self.exists(name):
raise ContainerNotExists("The container (%s) does not exist!" % name)
#
cmd = ['/usr/bin/lxc-info', '-n', name, "-H"]
out = subprocess.check_output(cmd).splitlines()
clean = []
info = {}
#
for line in out:
if line not in clean:
clean.append(line)
#
for line in clean:
key, value = line.split(":")
# strip
key = key.lstrip()
value = value.lstrip()
#
key = key.replace(" ", "_")
info[key.lower()] = value
# get container size
info['size'] = self.__get_container_size(name)
return info
def __get_container_size(self, name):
cmd = ['/usr/bin/du', '--total', '-s', '/var/lib/lxc/%s' % name]
out = subprocess.check_output(cmd).splitlines()
size = 0
for l in out:
key, value = l.split('\t')
if value == 'total':
size = key
return int(key)
def freeze(self, name):
"""
freezes the container
"""
if not self.exists(name):
raise ContainerNotExists("The container (%s) does not exist!" % name)
cmd = ['/usr/bin/lxc-freeze', '-n', name]
subprocess.check_call(cmd)
def unfreeze(self, name):
"""
unfreezes the container
"""
if not self.exists(name):
raise ContainerNotExists("The container (%s) does not exist!" % name)
cmd = ['lxc-unfreeze', '-n', name]
subprocess.check_call(cmd)
def notify(self, name, states, callback):
"""
executes the callback function with no parameters when the container reaches the specified state or states
states can be or-ed or and-ed
notify('test', 'STOPPED', letmeknow)
notify('test', 'STOPPED|RUNNING', letmeknow)
"""
if not self.exists(name):
raise ContainerNotExists("The container (%s) does not exist!" % name)
cmd = ['lxc-wait', '-n', name, '-s', states]
def th():
subprocess.check_call(cmd)
callback()
_logger.info("Waiting on states %s for container %s", states, name)
threading.Thread(target=th).start()
def checkconfig(self):
"""
returns the output of lxc-checkconfig
"""
cmd = ['lxc-checkconfig']
return subprocess.check_output(cmd).replace('[1;32m', '').replace('[1;33m', '').replace('[0;39m', '').replace('[1;32m', '').replace(' ', '').split('\n')
def create(self, name, config_file=None, template=None, backing_store=None, template_options=None):
"""
Create a new container
raises ContainerAlreadyExists exception if the container name is reserved already.
:param template_options: Options passed to the specified template
:type template_options: list or None
"""
if self.exists(name):
raise ContainerAlreadyExists("The Container %s is already created!" % name)
command = []
command.append("lxc-create -n %s" % name)
if config_file:
command.append(' -f %s' % config_file)
if template:
command.append(' -t %s' % template)
if backing_store:
command.append(' -B %s' % backing_store)
if template_options:
command.append(' -- %s' % template_options)
print " ".join(command)
print
# create = subprocess.check_call(command, shell=True)
create = subprocess.check_call(" ".join(command), shell=True)
print
print create
print
# if create == 0:
# if not self.exists(name):
# _logger.critical("The Container %s doesn't seem to be created! (options: %s)", name, command[3:])
# raise ContainerNotExists("The container (%s) does not exist!" % name)
#
# _logger.info("Container %s has been created with options %s", name, command[3:])
# return False
return True
def reset_password(self, container_name, username, password):
call = [
'echo',
'"%s:${PASSWORD:-%s}"' % (username, password),
"|",
"chroot",
"/var/lib/lxc/%s/rootfs/ chpasswd" % container_name
]
subprocess.check_call(call, shell=True)
# subprocess.call("echo \"ubuntu:${PASSWORD:-%(password)s}\" | chroot /var/lib/lxc/%(hostname)s/rootfs/ chpasswd" % task['parameters'], shell=True)
return True
# def running():
# '''
# returns a list of the currently running containers
# '''
# return all_as_dict()['Running']
# def stopped():
# '''
# returns a list of the stopped containers
# '''
# return all_as_dict()['Stopped']
# def all_as_dict():
# '''
# returns a dict {'Running': ['cont1', 'cont2'],
# 'Stopped': ['cont3', 'cont4']
# }
#
# '''
# cmd = ['lxc-ls']
# out = subprocess.check_output(cmd).splitlines()
# print out
# stopped = []
# running = []
# frozen = []
# current = None
# for c in out:
# c = c.strip()
# if c == 'RUNNING':
# current = running
# continue
# if c == 'STOPPED':
# current = stopped
# continue
# if c == 'FROZEN':
# current = frozen
# continue
# if not len(c):
# continue
# current.append(c)
# return {'Running': running,
# 'Stopped': stopped,
# 'Frozen': frozen}
# def all_as_list():
# '''
# returns a list of all defined containers
# '''
# as_dict = all_as_dict()
# containers = as_dict['Running'] + as_dict['Frozen'] + as_dict['Stopped']
# containers_list = []
# for i in containers:
# i = i.replace(' (auto)', '')
# containers_list.append(i)
# return containers_list
# def kill(name, signal):
# '''
# sends a kill signal to process 1 of ths container <name>
# :param signal: numeric signal
# '''
# if not exists(name):
# raise ContainerNotExists("The container (%s) does not exist!" % name)
# cmd = ['lxc-kill', '--name=%s' % name, signal]
# subprocess.check_call(cmd)
# def shutdown(name, wait=False, reboot=False):
# '''
# graceful shutdown sent to the container
# :param wait: should we wait for the shutdown to complete?
# :param reboot: reboot a container, ignores wait
# '''
# if not exists(name):
# raise ContainerNotExists("The container (%s) does not exist!" % name)
# cmd = ['lxc-shutdown', '-n', name]
# if wait:
# cmd += ['-w']
# if reboot:
# cmd += ['-r']
#
# subprocess.check_call(cmd)
# def monitor(name, callback):
# '''
# monitors actions on the specified container,
# callback is a function to be called on
# '''
# global _monitor
# if not exists(name):
# raise ContainerNotExists("The container (%s) does not exist!" % name)
# if _monitor:
# if _monitor.is_monitored(name):
# raise Exception("You are already monitoring this container (%s)" % name)
# else:
# _monitor = _LXCMonitor()
# logging.info("Starting LXC Monitor")
# _monitor.start()
# def kill_handler(sg, fr):
# stop_monitor()
# signal.signal(signal.SIGTERM, kill_handler)
# signal.signal(signal.SIGINT, kill_handler)
# _monitor.add_monitor(name, callback)
# def unmonitor(name):
# if not exists(name):
# raise ContainerNotExists("The container (%s) does not exist!" % name)
# if not _monitor:
# raise Exception("LXC Monitor is not started!")
# if not _monitor.is_monitored(name):
# raise Exception("This container (%s) is not monitored!" % name)
# _monitor.rm_monitor(name)
# def stop_monitor():
# global _monitor
# if _monitor:
# logging.info("Killing LXC Monitor")
# _monitor.kill()
# _monitor = None
# signal.signal(signal.SIGTERM, signal.SIG_DFL)
# signal.signal(signal.SIGINT, signal.SIG_DFL)

22
setup.py Normal file
View file

@ -0,0 +1,22 @@
# coding: utf-8
from setuptools import setup
setup(
name='SWSCloudNode',
version='2.0.1',
author='Vyacheslav Anzhiganov',
author_email='vanzhiganov@ya.ru',
packages=[
'SWSCloudNode',
'SWSCloudNode.lxc',
],
scripts=[
'cloud_node_agent.py',
'cloud_node_statistics.py',
],
package_data=[],
install_requires=[
'requests'
],
)