No OneTemporary
Actions

Size

37 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
	index dad0353..a450eed 100644
	--- a/.gitlab-ci.yml
	+++ b/.gitlab-ci.yml
	@@ -1,49 +1,49 @@

	stages:
	- unit-test
	- build

	default:
	image: quay.io/podman/stable

	.auth:
	script: &before
	- 'if [ -n "${CI_COMMIT_REF_SLUG}" ]; then sudo -u podman podman login -u "$REGISTRY_USER" --password-stdin "$REGISTRY" <<< "$REGISTRY_PASSWORD"; fi'
	- IMAGE_PREFIX="$REGISTRY/infra/lilybuild"
	- IMAGE_VER="${CI_COMMIT_REF_SLUG-none}"

	.push:
	script: &push
	- if [ -n "${CI_COMMIT_REF_SLUG}" ]; then sudo -u podman podman push "$IMAGE"; fi

	unit-test:
	stage: unit-test
	image: docker.io/buildbot/buildbot-master:v4.2.1
	script:
	- - /buildbot_venv/bin/pip3 install jsonschema
	+ - /buildbot_venv/bin/pip3 install jsonschema backports.tarfile
	- . /buildbot_venv/bin/activate
	- ./lilybuild/run-tests.sh

	build:master:
	stage: build
	script:
	- *before
	- IMAGE="$IMAGE_PREFIX/buildbot-master:$IMAGE_VER"
	- sudo -u podman ./build-master.sh -t "$IMAGE"
	- *push

	build:worker:
	stage: build
	script:
	- *before
	- IMAGE="$IMAGE_PREFIX/buildbot-worker:$IMAGE_VER"
	- sudo -u podman ./build-worker.sh -t "$IMAGE"
	- *push

	build:volume-helper:
	stage: build
	script:
	- *before
	- IMAGE="$IMAGE_PREFIX/volume-helper:$IMAGE_VER"
	- sudo -u podman ./build-volume-helper.sh -t "$IMAGE"
	- *push
	diff --git a/Containerfile.master b/Containerfile.master
	index 2a6aed9..8d83e5a 100644
	--- a/Containerfile.master
	+++ b/Containerfile.master
	@@ -1,4 +1,4 @@

	FROM docker.io/buildbot/buildbot-master:v4.2.1

	-RUN /buildbot_venv/bin/pip3 install jsonschema
	+RUN /buildbot_venv/bin/pip3 install jsonschema backports.tarfile
	diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
	index c7063bb..ace304b 100644
	--- a/lilybuild/lilybuild/ci_steps.py
	+++ b/lilybuild/lilybuild/ci_steps.py
	@@ -1,405 +1,427 @@

	from buildbot.plugins import *
	from buildbot.process import buildstep, logobserver
	from buildbot.interfaces import IRenderable
	from twisted.internet import defer
	from .ci_syntax import ci_file
	from .helpers import rsync_rules_from_artifacts, ci_vars_to_cmds
	import re
	+import sys

	def on_success(step):
	return step.build.results == util.SUCCESS

	DEFAULT_SCRIPT_HEADER = '''\
	#!/bin/sh

	set -e -x
	cd /build

	'''

	+def fill_list(*args):
	+ return list(args)
	+
	class RunCIJobStep(steps.BuildStep):
	# 200 MiB
	artifact_max_size = 200 * 1024 * 1024
	default_image = 'alpine'
	script_header = DEFAULT_SCRIPT_HEADER
	master_job_artifact_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/builds/%(prop:lilybuild_root_build_id)s/jobs/%(kw:job)s/artifacts'
	artifact_file_name = 'artifacts.tar'
	master_job_artifact_file_name_pattern = master_job_artifact_dir_pattern + '/' + artifact_file_name
	+ master_pages_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/pages'

	def __init__(
	self,
	lbc,
	src_relative=None,
	src_dir=None,
	storage_dir=None,
	repo_id=None,
	result_relative=None,
	result_dir=None,
	artifact_stage_relative=None,
	artifact_stage_dir=None,
	job_prop=None,
	**kwargs):
	self.lbc = lbc
	self.src_relative = src_relative
	self.src_dir = src_dir
	self.work_root_dir = kwargs['workdir']
	self.script_dir = 'script'
	self.storage_dir = storage_dir
	self.repo_id = repo_id
	self.artifact_stage_relative = artifact_stage_relative
	self.artifact_stage_dir = artifact_stage_dir
	self.result_relative = result_relative
	self.result_dir = result_dir
	super().__init__(name='Run step', **kwargs)

	@defer.inlineCallbacks
	def run(self):
	job_prop = self.getProperty('lilybuild_job_prop')
	job = ci_file.CIJob.from_prop(job_prop)
	job_index = self.getProperty('lilybuild_job_index')

	next_steps = yield self.job_to_steps(job, job_index)

	self.build.addStepsAfterCurrentStep(next_steps)
	return util.SUCCESS

	@defer.inlineCallbacks
	def get_ci_variables(self, job):
	res = {}
	res.update(job.get_predefined_ci_variables())
	res.update(self.getProperty('lilybuild_pipeline_vars'))
	res['CI_JOB_IMAGE'] = job.image or self.default_image
	res['CI_JOB_URL'] = yield self.build.getUrl()
	res['CI_JOB_ID'] = self.build.buildid
	res['CI_PROJECT_DIR'] = '/build'
	try:
	repo = self.lbc.repos[self.getProperty('lilybuild_repo_id')]
	variables = yield repo['variables_getter'](self.build)
	res_vars = {}
	for var in variables:
	value = variables[var]
	if IRenderable.providedBy(value):
	value = yield self.build.render(value)
	res_vars[var] = value
	res.update(res_vars)
	except Exception as e:
	self.addCompleteLog('exception', f'{e}')
	return res

	@defer.inlineCallbacks
	def job_to_steps(self, job, job_index):
	script_name = self.script_dir + '/run.sh'
	variables = yield self.get_ci_variables(job)
	var_cmds = ci_vars_to_cmds(variables)
	script_step = steps.StringDownload(
	self.script_header +
	'\n' + var_cmds + '\n\n' +
	'\n\n'.join(job.script),
	name='Set up script',
	workerdest=script_name,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	chmod_step = steps.ShellCommand(
	name='Make script executable',
	command=['chmod', '+x', script_name],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	artifact_steps = []
	dep_job_indices = self.getProperty('lilybuild_dependency_job_indices')
	if dep_job_indices:
	for i in dep_job_indices:
	download_job = steps.FileDownload(
	mastersrc=util.Interpolate(
	self.master_job_artifact_file_name_pattern,
	st=self.storage_dir,
	job=i,
	),
	maxsize=self.artifact_max_size,
	name=f'Download artifacts from job #{i}',
	workerdest=self.artifact_file_name,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	unarchive_job = steps.ShellCommand(
	name=f'Unarchive artifacts from job #{i}',
	command=[
	'tar',
	'-xf',
	self.artifact_file_name,
	'-C',
	self.src_relative,
	],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	artifact_steps += [download_job, unarchive_job]

	run_step = steps.ShellCommand(
	name='Run script in container',
	command=[
	'/lilybuild/podman-helper',
	job.image or self.default_image,
	self.src_relative,
	self.script_dir,
	self.result_relative,
	],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	clean_script_step = steps.ShellCommand(
	name='Clean script dir',
	command=[
	'rm',
	'-rf',
	self.script_dir,
	],
	workdir=self.work_root_dir,
	alwaysRun=True,
	)

	steps_to_run = [script_step, chmod_step] + artifact_steps + [run_step, clean_script_step]
	if 'paths' in job.artifacts:
	clean_stage_dir_step = steps.ShellCommand(
	name='Clean stage dir',
	command=[
	'rm',
	'-rf',
	self.artifact_stage_relative,
	],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)
	collect_artifact_step = steps.ShellCommand(
	name='Collect artifacts',
	command=[
	'rsync',
	'-av',
	self.result_relative + '/',
	'--delete',
	'--prune-empty-dirs',
	] + rsync_rules_from_artifacts(job.artifacts) + [
	self.artifact_stage_relative,
	],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)
	archive_artifact_step = steps.ShellCommand(
	name='Archive artifacts',
	command=[
	'tar',
	'-cf',
	self.artifact_file_name,
	'-C',
	self.artifact_stage_relative,
	'.',
	],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)
	masterdest = util.Interpolate(
	self.master_job_artifact_file_name_pattern,
	st=self.storage_dir,
	job=job_index,
	doStepIf=on_success,
	)
	upload_artifact_step = steps.FileUpload(
	workersrc=self.artifact_file_name,
	maxsize=self.artifact_max_size,
	name='Upload artifacts',
	masterdest=masterdest,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)
	steps_to_run += [clean_stage_dir_step, collect_artifact_step, archive_artifact_step, upload_artifact_step]
	clean_stage_dir_again_step = steps.ShellCommand(
	name='Clean stage, result and artifact',
	command=[
	'rm',
	'-rf',
	self.artifact_stage_relative,
	self.result_relative,
	self.artifact_file_name,
	],
	workdir=self.work_root_dir,
	alwaysRun=True,
	)
	steps_to_run.append(clean_stage_dir_again_step)

	+ if job.is_pages() and 'paths' in job.artifacts:
	+ deploy_pages_step = steps.MasterShellCommand(
	+ command=util.Transform(fill_list,
	+ sys.executable,
	+ '-m', 'lilybuild.pages',
	+ util.Interpolate(
	+ self.master_pages_dir_pattern,
	+ st=self.storage_dir,
	+ ),
	+ masterdest,
	+ ),
	+ name='Deploy pages',
	+ logEnviron=False,
	+ doStepIf=on_success,
	+ )
	+ steps_to_run.append(deploy_pages_step)
	+
	return steps_to_run

	class TriggerMultipleJobsStep(steps.Trigger):
	properties_to_keep = [
	'branch',
	'revision',
	'repository',
	'harbormaster_build_target_phid',
	'harbormaster_variable_buildable.diff',
	'harbormaster_variable_repository.staging.ref',
	'harbormaster_variable_repository.staging.uri',
	'harbormaster_variable_repository.uri',
	'lilybuild_repo',
	'lilybuild_repo_id',
	'lilybuild_pipeline_vars',
	]

	def __init__(self, lbc, jobs_with_data, **kwargs):
	self.lbc = lbc
	self.jobs = jobs_with_data
	super().__init__(schedulerNames=[self.lbc.triggerable_scheduler_name], **kwargs)

	def getSchedulersAndProperties(self):
	ret = []
	common_properties = {
	'lilybuild_root_build_id': self.build.buildid,
	}
	for prop in self.properties_to_keep:
	if self.hasProperty(prop):
	common_properties[prop] = self.getProperty(prop)

	for (job, i, dep_job_indices) in self.jobs:
	properties = common_properties.copy()
	properties['lilybuild_job_prop'] = job.to_prop()
	properties['lilybuild_job_index'] = i
	properties['virtual_builder_name'] = 'lilybuild-job - ' + common_properties['lilybuild_repo'] + ' - ' + job.name
	properties['lilybuild_dependency_job_indices'] = dep_job_indices
	ret.append({
	'sched_name': self.lbc.triggerable_scheduler_name,
	'props_to_set': properties,
	'unimportant': False,
	})

	return ret

	class AnalyzeCIFileCommand(buildstep.ShellMixin, steps.BuildStep):
	ci_def_file = '.gitlab-ci.yml'
	build_target_prop_name = 'harbormaster_build_target_phid'

	def __init__(
	self,
	lbc,
	src_relative=None,
	src_dir=None,
	storage_dir=None,
	repo_id=None,
	result_relative=None,
	result_dir=None,
	artifact_stage_relative=None,
	artifact_stage_dir=None,
	**kwargs):
	kwargs['name'] = 'Analyze CI file'
	kwargs['command'] = ['cat', self.ci_def_file]
	self.lbc = lbc
	self.src_relative = src_relative
	self.src_dir = src_dir
	self.work_root_dir = kwargs['workdir']
	self.script_dir = 'script'
	self.storage_dir = storage_dir
	self.repo_id = repo_id
	self.artifact_stage_relative = artifact_stage_relative
	self.artifact_stage_dir = artifact_stage_dir
	self.result_relative = result_relative
	self.result_dir = result_dir
	kwargs['workdir'] = self.src_dir
	kwargs = self.setupShellMixin(kwargs)
	super().__init__(**kwargs)
	self.observer = logobserver.BufferLogObserver()
	self.addLogObserver('stdio', self.observer)

	def stage_to_step(self, stage_name, stage_jobs, job_name_to_index_map, ci_file):
	jobs_with_data = []
	for job in stage_jobs:
	dep_job_names = [
	jn
	for jn in ci_file.get_jobs_to_pull_artifacts_from(job.name)
	if ci_file.jobs[jn].has_artifacts_archive()
	]
	dep_job_indices = [job_name_to_index_map[jn] for jn in dep_job_names]
	jobs_with_data.append((job, job_name_to_index_map[job.name], dep_job_indices))

	trigger = TriggerMultipleJobsStep(
	name=stage_name,
	lbc=self.lbc,
	jobs_with_data=jobs_with_data,
	waitForFinish=True,
	doStepIf=on_success,
	)

	return trigger

	def get_steps(self, stdout):
	f = ci_file.CIFile(stdout)

	stages = f.get_grouped_jobs()
	jobs = [job for (stage, js) in f.get_grouped_jobs() for job in js]
	job_names = [job.name for job in jobs]
	job_name_to_index_map = {}
	for (i, j) in enumerate(jobs):
	job_name_to_index_map[j.name] = i

	steps = [self.stage_to_step(stage_name, stage_jobs, job_name_to_index_map, f) for (stage_name, stage_jobs) in stages]
	print('steps:', steps)
	return steps

	def get_is_phorge(self):
	return not not self.getProperty(self.build_target_prop_name)

	def get_ref_and_type(self):
	ref_type = 'branch'
	ref = self.getProperty('branch')

	if self.getProperty('category') == 'tag':
	ref_type = 'tag'

	if ref is not None:
	m = re.match(r'^refs/(heads\|tags)/(.+)$', ref)
	if m:
	ref = m.group(2)

	return (ref, ref_type)

	@defer.inlineCallbacks
	def get_pipeline_ci_vars(self):
	url = yield self.build.getUrl()
	res = {
	'CI_PIPELINE_ID': self.build.buildid,
	'CI_PIPELINE_IID': self.build.buildid,
	'CI_PIPELINE_URL': url,
	'CI_PROJECT_ID': self.getProperty('lilybuild_repo_id'),
	'CI_CONFIG_PATH': self.ci_def_file,
	}
	if not self.get_is_phorge():
	res['CI_COMMIT_SHA'] = self.getProperty('got_revision')
	res['CI_COMMIT_SHORT_SHA'] = res['CI_COMMIT_SHA'][:8]
	(ref, ref_type) = self.get_ref_and_type()
	res['CI_COMMIT_REF_NAME'] = ref
	res['CI_COMMIT_REF_SLUG'] = ci_file.ci_slugify(ref)
	res['CI_COMMIT_REF_PROTECTED'] = 'false'
	if ref_type == 'tag':
	res['CI_COMMIT_TAG'] = ref
	elif ref_type == 'branch':
	res['CI_COMMIT_BRANCH'] = ref
	return res

	@defer.inlineCallbacks
	def run(self):
	# run './build.sh --list-stages' to generate the list of stages
	cmd = yield self.makeRemoteShellCommand()
	yield self.runCommand(cmd)

	# if the command passes extract the list of stages
	result = cmd.results()
	if result == util.SUCCESS:
	pipeline_vars = yield self.get_pipeline_ci_vars()
	self.setProperty('lilybuild_pipeline_vars', pipeline_vars, self.__class__.__name__)
	# create a ShellCommand for each stage and add them to the build
	self.build.addStepsAfterCurrentStep(self.get_steps(self.observer.getStdout()))

	return result
	diff --git a/lilybuild/lilybuild/ci_syntax/ci_file.py b/lilybuild/lilybuild/ci_syntax/ci_file.py
	index 539ab08..d052b69 100644
	--- a/lilybuild/lilybuild/ci_syntax/ci_file.py
	+++ b/lilybuild/lilybuild/ci_syntax/ci_file.py
	@@ -1,177 +1,180 @@

	import yaml
	import jsonschema
	import json
	import os
	import re

	# https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/editor/schema/ci.json
	schema_file = os.path.join(os.path.dirname(__file__), 'ci.json')
	schema = None

	def get_schema():
	global schema
	if schema is not None:
	return schema
	else:
	with open(schema_file, 'r') as f:
	schema = json.loads(f.read())
	return schema

	def toplevel_entries():
	return get_schema()['properties']

	def normalize_script(script):
	if script is None:
	return []
	elif isinstance(script, str):
	return [script]
	else:
	res = []
	for l in script:
	if isinstance(l, str):
	res.append(l)
	else:
	res += l
	return res

	slug_re = re.compile('[^0-9a-z]')
	def ci_slugify(s):
	return re.sub(slug_re, '-', s.lower()[:63]).strip('-')

	class CIJob:
	def __init__(self, job_name, job_stage, job_struct, defaults):
	self.name = job_name
	self.stage = job_stage
	self.struct_raw = job_struct
	self.defaults_raw = defaults
	self.image = job_struct.get('image', defaults.get('image'))
	self.before_script = normalize_script(job_struct.get('before_script', defaults.get('before_script')))
	self.script = normalize_script(job_struct.get('script', defaults.get('script')))
	self.after_script = normalize_script(job_struct.get('after_script', defaults.get('after_script')))
	self.artifacts = job_struct.get('artifacts', defaults.get('artifacts', {}))
	self.dependencies = job_struct.get('dependencies')

	def get_predefined_ci_variables(self):
	return {
	'CI': 'true',
	'CI_JOB_NAME': self.name,
	'CI_JOB_NAME_SLUG': ci_slugify(self.name),
	'CI_JOB_STAGE': self.stage,
	}

	def has_artifacts_archive(self):
	return self.artifacts and 'paths' in self.artifacts

	def to_prop(self):
	return {
	'name': self.name,
	'stage': self.stage,
	'struct_raw': self.struct_raw,
	'defaults_raw': self.defaults_raw,
	}

	@classmethod
	def from_prop(cls, prop):
	return cls(
	prop['name'],
	prop['stage'],
	prop['struct_raw'],
	prop['defaults_raw']
	)

	+ def is_pages(self):
	+ return self.name == 'pages' or self.struct_raw.get('pages', False)
	+
	OLD_TOPLEVEL_DEFAULTS = ['image', 'services', 'cache', 'before_script', 'after_script']
	DEFAULT_STAGES = ['.pre', 'build', 'test', 'deploy', '.post']
	DEFAULT_JOB_STAGE = 'test'

	class CIValidationError(Exception):
	pass

	class CIFile:
	'''
	Class for parsing CI file.
	'''

	def __init__(self, file_content):
	'''
	Construct a CI File from its text content.
	'''
	f = yaml.safe_load(file_content)
	if f is None:
	self.stages = []
	self.jobs = {}
	return

	jsonschema.validate(instance=f, schema=get_schema())

	self.stages = f.get('stages', DEFAULT_STAGES)
	self.jobs = {}

	defaults = f.get('default', {})
	if f.get('variables'):
	defaults['variables'] = f['variables']
	for kw in OLD_TOPLEVEL_DEFAULTS:
	if kw not in defaults and kw in f:
	defaults[kw] = f[kw]

	for job_name, job_struct in f.items():
	# 'pages' is a special job
	if job_name != 'pages' and job_name in toplevel_entries():
	continue
	# jobs starting with . will only be used for base jobs of other jobs,
	# they themselves are not run
	if job_name.startswith('.'):
	continue
	job_stage = job_struct.get('stage', DEFAULT_JOB_STAGE)
	if job_stage not in self.stages:
	raise CIValidationError(f'Job "{job_name}": Stage "{job_stage}" is not specified in CI file')
	self.jobs[job_name] = CIJob(job_name, job_stage, job_struct, defaults)

	self.validate_logic()

	def validate_logic(self):
	for job_name in self.jobs:
	self.validate_job(job_name)

	def validate_job(self, job_name):
	j = self.jobs[job_name]
	my_stage_order = self.stage_order(job_name)
	if j.dependencies is not None:
	for d in j.dependencies:
	try:
	stage_order = self.stage_order(d)
	except KeyError:
	raise CIValidationError(f'Dependency "{d}" of job "{job_name}" does not exist')
	if stage_order >= my_stage_order:
	raise CIValidationError(f'Dependency "{d}" of job "{job_name}" is not before the job in stage')

	def stage_order(self, jn):
	return self.stages.index(self.jobs[jn].stage)

	def get_grouped_jobs(self):
	groups = {}
	for job_name in self.jobs:
	job = self.jobs[job_name]
	if job.stage not in groups:
	groups[job.stage] = []
	groups[job.stage].append(job)
	res = []
	for stage in self.stages:
	if stage in groups:
	res.append((stage, groups[stage]))
	return res

	def get_jobs_to_pull_artifacts_from(self, job_name):
	'''
	Get a list of names of jobs of which the artifacts will be pulled
	from for the job named `job_name`.
	'''
	dependencies = self.jobs[job_name].dependencies
	if dependencies is None:
	dependencies = []
	cur_job_stage_order = self.stage_order(job_name)
	for jn in self.jobs:
	so = self.stage_order(jn)
	if so < cur_job_stage_order:
	dependencies.append(jn)
	return dependencies
	diff --git a/lilybuild/lilybuild/pages.py b/lilybuild/lilybuild/pages.py
	new file mode 100644
	index 0000000..4b01e02
	--- /dev/null
	+++ b/lilybuild/lilybuild/pages.py
	@@ -0,0 +1,84 @@
	+
	+import os
	+import subprocess
	+import shutil
	+import sys
	+try:
	+ import tarfile
	+ tarfile.FilterError
	+except AttributeError:
	+ import backports.tarfile as tarfile
	+
	+'''
	+This module implements blue-green deployment for pages.
	+'''
	+
	+FIRST = '0'
	+SECOND = '1'
	+CURRENT = 'cur'
	+
	+def init_deployment(dir_name):
	+ '''
	+ Init the deployment at `dir_name` and return which directory we should
	+ write into for the next deployment.
	+ '''
	+ os.makedirs(os.path.join(dir_name, FIRST), exist_ok=True)
	+ os.makedirs(os.path.join(dir_name, SECOND), exist_ok=True)
	+ link_file = os.path.join(dir_name, CURRENT)
	+ try:
	+ current = os.readlink(link_file)
	+ except FileNotFoundError as e:
	+ os.symlink(SECOND, link_file, target_is_directory=True)
	+ current = SECOND
	+ return FIRST if current == SECOND else SECOND
	+
	+def switch_symlink(dir_name, target):
	+ subprocess.run([
	+ 'ln',
	+ '-sfvn',
	+ '--',
	+ target,
	+ os.path.join(dir_name, CURRENT),
	+ ], check=True)
	+
	+def empty_directory(dir_name):
	+ shutil.rmtree(dir_name)
	+ os.makedirs(dir_name, exist_ok=True)
	+
	+class TarPagesFilter:
	+ public_dir = 'public'
	+ def __init__(
	+ self,
	+ limit_bytes=10010241024 # 100 MiB
	+ ):
	+ self.total_bytes_extracted = 0
	+ self.limit_bytes = limit_bytes
	+
	+ def __call__(self, member, path):
	+ filtered_member = tarfile.data_filter(member, path)
	+ if not filtered_member:
	+ return None
	+ if self.total_bytes_extracted + filtered_member.size > self.limit_bytes:
	+ self.total_bytes_extracted = self.limit_bytes + 1
	+ raise RuntimeError('Limit exceeded')
	+ name = os.path.normpath(filtered_member.name)
	+ if not (name == self.public_dir or name.startswith(self.public_dir + '/')):
	+ return None
	+
	+ self.total_bytes_extracted += filtered_member.size
	+ return filtered_member
	+
	+def extract_archive(dir_name, archive_file):
	+ with tarfile.open(archive_file) as tf:
	+ tf.errorlevel = 1
	+ tf.extractall(dir_name, filter=TarPagesFilter())
	+
	+def deploy_pages(dir_name, archive_file):
	+ target = init_deployment(dir_name)
	+ full_target = os.path.join(dir_name, target)
	+ empty_directory(full_target)
	+ extract_archive(full_target, archive_file)
	+ switch_symlink(dir_name, target)
	+
	+if __name__ == '__main__':
	+ deploy_pages(sys.argv[1], sys.argv[2])
	diff --git a/lilybuild/lilybuild/tests/ci_syntax/ci_file_test.py b/lilybuild/lilybuild/tests/ci_syntax/ci_file_test.py
	index 0c3228a..8997573 100644
	--- a/lilybuild/lilybuild/tests/ci_syntax/ci_file_test.py
	+++ b/lilybuild/lilybuild/tests/ci_syntax/ci_file_test.py
	@@ -1,109 +1,115 @@

	import unittest
	import yaml
	from lilybuild.ci_syntax.ci_file import CIFile, CIValidationError
	import os

	self_dir = os.path.dirname(__file__)
	res_dir = os.path.join(self_dir, 'res')
	def get_res(name):
	with open(os.path.join(res_dir, name + '.yaml'), 'r') as f:
	return f.read()

	class CIFileTest(unittest.TestCase):
	def test_empty(self):
	r = CIFile('')
	self.assertEqual(r.stages, [])
	self.assertEqual(r.jobs, {})

	def test_invalid(self):
	with self.assertRaises(yaml.composer.ComposerError) as m:
	r = CIFile('*xxx')

	def test_simple(self):
	r = CIFile(get_res('simple'))
	self.assertEqual(r.stages, ['build'])
	self.assertEqual(list(r.jobs), ['build-a'])
	self.assertEqual(r.jobs['build-a'].image, 'test')
	self.assertEqual(r.jobs['build-a'].before_script, ['ls'])
	self.assertEqual(r.jobs['build-a'].script, ['make', 'make install'])
	self.assertEqual(r.jobs['build-a'].after_script, ['find', 'echo ok'])

	def test_noscript(self):
	r = CIFile(get_res('noscript'))

	self.assertEqual(r.stages, ['build'])
	self.assertEqual(list(r.jobs), ['build-a'])
	self.assertEqual(r.jobs['build-a'].image, 'test')
	self.assertEqual(r.jobs['build-a'].script, [])

	def test_defaults(self):
	r = CIFile(get_res('defaults'))
	self.assertEqual(r.stages, ['build'])
	self.assertEqual(list(r.jobs), ['build-a'])
	self.assertEqual(r.jobs['build-a'].image, 'test')
	self.assertEqual(r.jobs['build-a'].before_script, ['ls'])
	self.assertEqual(r.jobs['build-a'].script, ['make', 'make install'])
	self.assertEqual(r.jobs['build-a'].after_script, ['find', 'echo ok'])

	def test_pages(self):
	r = CIFile(get_res('pages'))
	self.assertEqual(r.stages, ['test'])
	self.assertEqual(list(r.jobs), ['pages'])
	self.assertEqual(r.jobs['pages'].script, ['make docs'])
	+ self.assertTrue(r.jobs['pages'].is_pages())
	+
	+ def test_pages_attr(self):
	+ r = CIFile(get_res('pages_attr'))
	+ self.assertFalse(r.jobs['not-pages'].is_pages())
	+ self.assertTrue(r.jobs['is-pages'].is_pages())

	def test_dotjobs(self):
	r = CIFile(get_res('dotjobs'))
	self.assertEqual(list(r.jobs), [])

	def test_no_such_stage(self):
	with self.assertRaises(CIValidationError) as m:
	r = CIFile(get_res('no_such_stage'))

	def test_default_stages(self):
	r = CIFile(get_res('default_stages'))
	self.assertEqual(r.jobs['a'].stage, 'build')
	self.assertEqual(r.jobs['b'].stage, 'test')
	self.assertEqual(r.jobs['c'].stage, 'deploy')
	self.assertEqual(r.jobs['d'].stage, '.pre')
	self.assertEqual(r.jobs['e'].stage, '.post')
	self.assertEqual(r.jobs['f'].stage, 'test')

	def test_group_jobs(self):
	r = CIFile(get_res('group_jobs'))
	groups = r.get_grouped_jobs()
	self.assertEqual(len(groups), 2)
	self.assertEqual(groups[0][0], 'build')
	self.assertEqual(len(groups[0][1]), 2)
	self.assertTrue(next(j for j in groups[0][1] if j.name == 'a'))
	self.assertTrue(next(j for j in groups[0][1] if j.name == 'b'))
	self.assertEqual(groups[1][0], 'test')
	self.assertEqual(len(groups[1][1]), 2)
	self.assertTrue(next(j for j in groups[1][1] if j.name == 'c'))
	self.assertTrue(next(j for j in groups[1][1] if j.name == 'd'))

	def test_dependencies(self):
	r = CIFile(get_res('dependencies'))
	self.assertEqual(r.get_jobs_to_pull_artifacts_from('a'), [])
	self.assertEqual(r.get_jobs_to_pull_artifacts_from('a2'), [])
	self.assertEqual(r.get_jobs_to_pull_artifacts_from('b'), ['a'])
	self.assertEqual(set(r.get_jobs_to_pull_artifacts_from('b2')), set(['a', 'a2']))

	def test_nonexistent_dep(self):
	with self.assertRaises(CIValidationError) as m:
	r = CIFile(get_res('nonexistent_dep'))
	self.assertEqual(str(m.exception), 'Dependency "a3" of job "b" does not exist')

	def test_late_dep(self):
	with self.assertRaises(CIValidationError) as m:
	r = CIFile(get_res('late_dep'))
	self.assertEqual(str(m.exception), 'Dependency "a2" of job "a" is not before the job in stage')

	def test_has_artifacts_archive(self):
	r = CIFile(get_res('has_artifacts_archive'))
	self.assertTrue(r.jobs['has_archive'].has_artifacts_archive())
	self.assertFalse(r.jobs['no_archive'].has_artifacts_archive())
	self.assertFalse(r.jobs['no_artifacts'].has_artifacts_archive())

	if __name__ == '__main__':
	unittest.main()
	diff --git a/lilybuild/lilybuild/tests/ci_syntax/res/pages_attr.yaml b/lilybuild/lilybuild/tests/ci_syntax/res/pages_attr.yaml
	new file mode 100644
	index 0000000..45ad8c5
	--- /dev/null
	+++ b/lilybuild/lilybuild/tests/ci_syntax/res/pages_attr.yaml
	@@ -0,0 +1,12 @@
	+
	+stages:
	+ - test
	+
	+not-pages:
	+ image: node
	+ script: make docs
	+
	+is-pages:
	+ image: node
	+ script: make docs
	+ pages: true
	diff --git a/lilybuild/lilybuild/tests/pages_test.py b/lilybuild/lilybuild/tests/pages_test.py
	new file mode 100644
	index 0000000..a40bbd5
	--- /dev/null
	+++ b/lilybuild/lilybuild/tests/pages_test.py
	@@ -0,0 +1,128 @@
	+
	+import unittest
	+import tempfile
	+import os
	+import stat
	+import subprocess
	+from lilybuild.pages import (
	+ FIRST, SECOND, CURRENT, init_deployment,
	+ switch_symlink, extract_archive, deploy_pages,
	+)
	+
	+try:
	+ import tarfile
	+ tarfile.FilterError
	+except AttributeError:
	+ import backports.tarfile as tarfile
	+
	+
	+def is_dir(filename):
	+ return stat.S_ISDIR(os.lstat(filename).st_mode)
	+
	+def make_artifact_archive(root_dir):
	+ os.makedirs(os.path.join(root_dir, 'public'))
	+ with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
	+ print('test', file=f)
	+ os.makedirs(os.path.join(root_dir, 'other'))
	+ with open(os.path.join(root_dir, 'other', 'a'), 'w') as f:
	+ print('should not be there', file=f)
	+ archive = os.path.join(root_dir, 'artifacts.tar')
	+ with tarfile.open(archive, 'w') as f:
	+ f.add(os.path.join(root_dir, 'public'), 'public')
	+ f.add(os.path.join(root_dir, 'other'), 'other')
	+ return archive
	+
	+def make_artifact_archive2(root_dir):
	+ os.makedirs(os.path.join(root_dir, 'public'))
	+ with open(os.path.join(root_dir, 'public', 'b'), 'w') as f:
	+ print('test b', file=f)
	+ archive = os.path.join(root_dir, 'artifacts.tar')
	+ with tarfile.open(archive, 'w') as f:
	+ f.add(os.path.join(root_dir, 'public'), 'public')
	+ return archive
	+
	+def make_bad_artifact_archive(root_dir):
	+ os.makedirs(os.path.join(root_dir, 'public'))
	+ with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
	+ print('test', file=f)
	+ os.makedirs(os.path.join(root_dir, 'other'))
	+ with open(os.path.join(root_dir, 'other', 'a'), 'w') as f:
	+ print('should not be there', file=f)
	+ archive = os.path.join(root_dir, 'artifacts.tar')
	+ with tarfile.open(archive, 'w') as f:
	+ f.add(os.path.join(root_dir, 'public'), 'public')
	+ f.add(os.path.join(root_dir, 'other'), '../../../other')
	+ return archive
	+
	+class PagesTest(unittest.TestCase):
	+ def test_init_deployment(self):
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ res = init_deployment(dir_name)
	+ self.assertEqual(res, FIRST)
	+ self.assertTrue(is_dir(os.path.join(dir_name, FIRST)))
	+ self.assertTrue(is_dir(os.path.join(dir_name, SECOND)))
	+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)
	+
	+ res = init_deployment(dir_name)
	+ self.assertEqual(res, FIRST)
	+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)
	+
	+ def test_switch_symlink(self):
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ res = init_deployment(dir_name)
	+ switch_symlink(dir_name, res)
	+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), FIRST)
	+
	+ def test_extract_archive(self):
	+ with tempfile.TemporaryDirectory() as root_dir:
	+ archive_file = make_artifact_archive(root_dir)
	+ dir_name = os.path.join(root_dir, 'deployment')
	+ extract_archive(dir_name, archive_file)
	+ self.assertTrue(is_dir(os.path.join(dir_name, 'public')))
	+ self.assertTrue(os.path.exists(os.path.join(dir_name, 'public', 'a')))
	+ self.assertFalse(os.path.exists(os.path.join(dir_name, 'other')))
	+
	+ def test_extract_bad_archive(self):
	+ with tempfile.TemporaryDirectory() as root_dir:
	+ archive_file = make_bad_artifact_archive(root_dir)
	+ dir_name = os.path.join(root_dir, 'deployment')
	+ with self.assertRaises(tarfile.FilterError):
	+ extract_archive(dir_name, archive_file)
	+
	+ def test_deploy_pages(self):
	+ with tempfile.TemporaryDirectory() as root_dir:
	+ archive_file = make_artifact_archive(os.path.join(root_dir, 'a1'))
	+ bad_archive = make_bad_artifact_archive(os.path.join(root_dir, 'bad'))
	+ archive_file2 = make_artifact_archive2(os.path.join(root_dir, 'a2'))
	+ dir_name = os.path.join(root_dir, 'deployment')
	+ deploy_pages(dir_name, archive_file)
	+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), FIRST)
	+ self.assertTrue(os.path.exists(os.path.join(dir_name, FIRST, 'public', 'a')))
	+ with open(os.path.join(dir_name, FIRST, 'public', 'a')) as f:
	+ self.assertEqual(f.read(), 'test\n')
	+
	+ # deploy the same thing again, it should go to SECOND
	+ deploy_pages(dir_name, archive_file)
	+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)
	+ self.assertTrue(os.path.exists(os.path.join(dir_name, SECOND, 'public', 'a')))
	+ with open(os.path.join(dir_name, SECOND, 'public', 'a')) as f:
	+ self.assertEqual(f.read(), 'test\n')
	+
	+ # try to deploy the bad archive, it should not update the link
	+ with self.assertRaises(tarfile.FilterError):
	+ deploy_pages(dir_name, bad_archive)
	+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)
	+
	+ # deploy something else, verify it cleans up all old files
	+ deploy_pages(dir_name, archive_file2)
	+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), FIRST)
	+ self.assertTrue(os.path.exists(os.path.join(dir_name, FIRST, 'public', 'b')))
	+ self.assertFalse(os.path.exists(os.path.join(dir_name, FIRST, 'public', 'a')))
	+ self.assertTrue(os.path.exists(os.path.join(dir_name, SECOND, 'public', 'a')))
	+ self.assertFalse(os.path.exists(os.path.join(dir_name, SECOND, 'public', 'b')))
	+ with open(os.path.join(dir_name, FIRST, 'public', 'b')) as f:
	+ self.assertEqual(f.read(), 'test b\n')
	+
	+
	+if __name__ == '__main__':
	+ unittest.main()

File Metadata

Mime Type: text/x-diff
Expires: Fri, Oct 9, 4:35 AM (1 d, 8 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1784448
Default Alt Text: (37 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions