phorge.py
No OneTemporary
Actions

Size

10 KB

Referenced Files

None

Subscribers

None

phorge.py
View Options


	import stat
	import os
	import json
	from buildbot.plugins import *
	from buildbot.process import buildstep, logobserver
	from buildbot.interfaces import IRenderable
	from buildbot.steps.download_secret_to_worker import DownloadSecretsToWorker, RemoveWorkerFileSecret
	from buildbot.steps.worker import CompositeStepMixin
	from buildbot.util import bytes2unicode
	from twisted.internet import defer
	from twisted.python import log

	class PhorgeMixin(CompositeStepMixin, buildstep.ShellMixin):
	'''
	Should be inherited by a BuildStep.

	Required properties:
	self.lbc: LilyBuildConfig
	self.secret_dir: str
	'''
	staging_uri_prop_name = 'harbormaster_variable_repository.staging.uri'
	staging_ref_prop_name = 'harbormaster_variable_repository.staging.ref'
	diff_id_prop_name = 'harbormaster_variable_buildable.diff'
	build_target_prop_name = 'harbormaster_build_target_phid'
	arc_command = '/tools/arcanist/bin/arc'

	def setup_phorge_mixin(self, kwargs):
	shell_mixin_kwargs = {}
	shell_mixin_inherit_args = ['workdir']
	for a in shell_mixin_inherit_args:
	if a in kwargs:
	shell_mixin_kwargs[a] = kwargs[a]

	self.setupShellMixin(shell_mixin_kwargs)

	def get_is_phorge(self):
	return not not self.getProperty(self.build_target_prop_name)

	def get_secret_dir(self):
	# Taken from buildbot.util.git
	workerbuilddir = bytes2unicode(self.build.builder.config.workerbuilddir)
	workdir = self.workdir.rstrip('/\\')

	if os.path.isabs(workdir):
	parent_path = os.path.dirname(workdir)
	else:
	assert self.worker is not None
	parent_path = os.path.join(
	self.worker.worker_basedir, os.path.dirname(workdir)
	)

	basename = f'.{workerbuilddir}.{os.path.basename(workdir)}.lilybuild-phorge'
	secret_dir = os.path.join(parent_path, basename)
	log.msg('Arc secret dir is', secret_dir)
	return secret_dir

	@defer.inlineCallbacks
	def make_arc_command(self, args, **kwargs):
	cmd = yield self.makeRemoteShellCommand(
	command=[self.arc_command] + args,
	env={'HOME': self.get_secret_dir()},
	**kwargs
	)
	return cmd

	@defer.inlineCallbacks
	def run_arc_with_secret(self, args, **kwargs):
	try:
	res = yield self.download_arc_secret()
	if res != util.SUCCESS:
	return res
	cmd = yield self.make_arc_command(args, **kwargs)
	yield self.runCommand(cmd)
	return cmd.results()
	except Exception as e:
	raise e
	finally:
	yield self.remove_arc_secret()


	@defer.inlineCallbacks
	def download_arc_secret(self):
	arcrc = self.lbc.get_arcrc_for_repo(self.getProperty('lilybuild_repo_id'))
	if not arcrc:
	self.addCompleteLog('error', 'arcrc secret is not specified for the repository')
	return util.FAILURE

	arcrc_content = yield self.build.render(arcrc)

	path = os.path.join(self.get_secret_dir(), '.arcrc')

	yield self.downloadFileContentToWorker(
	path, arcrc_content, mode=stat.S_IRUSR \| stat.S_IWUSR, workdir=self.workdir
	)
	# If failed, it will raise
	return util.SUCCESS

	@defer.inlineCallbacks
	def remove_arc_secret(self):
	path = self.get_secret_dir()

	yield self.runRmdir(path, abandonOnFailure=False)
	return util.SUCCESS

	class CheckoutSourceFromPhorge(PhorgeMixin, steps.Git):
	'''
	Set a property `lilybuild_source`:
	`non-phorge` if the source is not a Differential Diff,
	`staging` if the source is checked out from staging area,
	`arc-patch` if the source is checked out by `arc patch`, either because
	there is no staging area defined, or because the staging area does not
	have the relevant diffs.
	'''
	def __init__(self, lbc, **kwargs):
	self.lbc = lbc
	self.setup_phorge_mixin(kwargs)
	super().__init__(**kwargs)

	@defer.inlineCallbacks
	def run_vc(self, branch, revision, patch):
	def set_prop(k, v):
	self.setProperty(k, v, self.__class__.__name__)

	is_phorge = self.get_is_phorge()
	if not is_phorge:
	set_prop('lilybuild_source', 'non-phorge')
	res = yield super().run_vc(branch, revision, patch)
	return res

	staging_uri = self.getProperty(self.staging_uri_prop_name)
	staging_ref = self.getProperty(self.staging_ref_prop_name)

	if staging_uri:
	old_repourl = self.repourl
	self.repourl = staging_uri
	# Assume branch is the same as staging ref
	try:
	res = yield super().run_vc(branch, revision, patch)
	if res == util.SUCCESS:
	# If we can get the source from the staging area, then we are done
	set_prop('lilybuild_source', 'staging')
	return res
	except:
	pass
	self.addCompleteLog('log', 'Cannot fetch source from staging area, trying to `arc patch`')
	self.repourl = old_repourl
	else:
	self.addCompleteLog('log', 'No staging area defined, trying to `arc patch`')

	res = yield super().run_vc(branch='HEAD', revision=None, patch=None)
	if res != util.SUCCESS:
	self.addCompleteLog('error', 'Cannot checkout repository head, unable to proceed')
	return res

	diff_id = self.getProperty(self.diff_id_prop_name)
	if not diff_id:
	self.addCompleteLog('error', 'Diff id is not present')
	return util.FAILURE

	res = yield self.run_arc_with_secret([
	'patch',
	'--diff', diff_id,
	'--nobranch', '--nocommit',
	])
	set_prop('lilybuild_source', 'arc-patch')
	return res

	class SendArtifactLinkToPhorge(PhorgeMixin, steps.BuildStep):
	def __init__(self, lbc, **kwargs):
	self.lbc = lbc
	self.setup_phorge_mixin(kwargs)
	super().__init__(
	name='Send artifact link to Phorge',
	doStepIf=lambda step: step.get_is_phorge(),
	alwaysRun=True,
	**kwargs
	)

	@defer.inlineCallbacks
	def run(self):
	if self.getProperty('lilybuild_require_approval'):
	# Do not send another artifact if this requires approval
	return util.SKIPPED

	build_url = yield self.build.getUrl()

	data = {
	'buildTargetPHID': self.getProperty(self.build_target_prop_name),
	'artifactKey': f'lilybuild-{self.build.buildid}',
	'artifactType': 'uri',
	'artifactData': {
	'uri': build_url,
	'name': f'Buildbot build #{self.build.buildid}',
	'ui.external': True,
	},
	}
	encoded_data = json.dumps(data)
	res = yield self.run_arc_with_secret([
	'call-conduit',
	'--',
	'harbormaster.createartifact',
	], initialStdin=encoded_data)
	return res

	class MaybeRequireApprovalForPhorge(PhorgeMixin, steps.BuildStep):
	def __init__(self, lbc, sources_need_approval=None, **kwargs):
	self.lbc = lbc
	if sources_need_approval is None:
	sources_need_approval = ['arc-patch']
	self.sources_need_approval = sources_need_approval
	self.setup_phorge_mixin(kwargs)
	super().__init__(
	name='Maybe require approval for phorge sources',
	doStepIf=lambda step: step.get_is_phorge(),
	alwaysRun=True,
	**kwargs
	)

	@defer.inlineCallbacks
	def run(self):
	def set_prop(k, v):
	self.setProperty(k, v, self.__class__.__name__)

	if self.getProperty('lilybuild_source') not in self.sources_need_approval:
	set_prop('lilybuild_require_approval', False)
	return util.SUCCESS

	build_data = yield self.master.data.get(('builds', self.build.buildid))
	build_request_data = yield self.master.data.get(('buildrequests', build_data['buildrequestid']))
	buildset_data = yield self.master.data.get(('buildsets', build_request_data['buildsetid']))
	rebuilt_buildid = buildset_data['rebuilt_buildid']
	if rebuilt_buildid is not None:
	# This is rebuilt, because someone approved it earlier
	set_prop('lilybuild_require_approval', False)
	return util.SUCCESS

	self.addCompleteLog('info', 'This build requires approval. To approve, rebuild this build.')
	build_url = yield self.build.getUrl()

	data = {
	'buildTargetPHID': self.getProperty(self.build_target_prop_name),
	'artifactKey': f'lilybuild-{self.build.buildid}-pending',
	'artifactType': 'uri',
	'artifactData': {
	'uri': build_url,
	'name': f'Requires approval',
	'ui.external': True,
	},
	}
	encoded_data = json.dumps(data)
	res = yield self.run_arc_with_secret([
	'call-conduit',
	'--',
	'harbormaster.createartifact',
	], initialStdin=encoded_data)
	set_prop('lilybuild_require_approval', True)
	return util.FAILURE


	class SendBuildStatusToPhorge(PhorgeMixin, steps.BuildStep):
	def __init__(self, lbc, **kwargs):
	self.lbc = lbc
	self.setup_phorge_mixin(kwargs)
	super().__init__(
	name='Send build status to Phorge',
	doStepIf=lambda step: step.get_is_phorge(),
	alwaysRun=True,
	**kwargs
	)

	@defer.inlineCallbacks
	def run(self):
	if self.getProperty('lilybuild_require_approval'):
	# Do not send final build result if this is skipped, because
	# otherwise we cannot update it later.
	return util.SKIPPED

	build_url = yield self.build.getUrl()

	data = {
	'receiver': self.getProperty(self.build_target_prop_name),
	'type': 'pass' if self.build.results == util.SUCCESS else 'fail',
	}
	encoded_data = json.dumps(data)
	res = yield self.run_arc_with_secret([
	'call-conduit',
	'--',
	'harbormaster.sendmessage',
	], initialStdin=encoded_data)
	return res

File Metadata

Mime Type: text/x-python
Expires: Fri, Nov 14, 10:43 PM (18 h, 6 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 632883
Default Alt Text: phorge.py (10 KB)

phorge.pyNo OneTemporaryActions

phorge.pyView Options

File Metadata

Event Timeline

phorge.py
No OneTemporary
Actions

phorge.py
View Options