Page MenuHomePhorge
Files F83724637

D305.1779953669.diff
No OneTemporary
Actions

Size
6 KB
Referenced Files
None
Subscribers
None

D305.1779953669.diff
View Options

diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
--- a/lilybuild/lilybuild/ci_steps.py
+++ b/lilybuild/lilybuild/ci_steps.py
@@ -190,6 +190,7 @@
st=self.storage_dir,
),
masterdest,
+ str(self.get_cur_repo_config()['pages_uncompressed_limit']),
),
name='Deploy pages',
logEnviron=False,
diff --git a/lilybuild/lilybuild/config.py b/lilybuild/lilybuild/config.py
--- a/lilybuild/lilybuild/config.py
+++ b/lilybuild/lilybuild/config.py
@@ -40,7 +40,7 @@
triggerable_scheduler_name = 'lilybuild-triggerable'
force_triggerable_scheduler_name = 'lilybuild-force-triggerable'
- def __init__(self, c, workernames, reports=None, phorge_base_url=None, phorge_token=None, ssh_priv_key=None, ssh_known_hosts=None, storage_dir=default_storage_dir, artifact_link_base=None, artifact_uncompressed_limit=None, artifact_compressed_limit=None):
+ def __init__(self, c, workernames, reports=None, phorge_base_url=None, phorge_token=None, ssh_priv_key=None, ssh_known_hosts=None, storage_dir=default_storage_dir, artifact_link_base=None, artifact_uncompressed_limit=None, artifact_compressed_limit=None, pages_uncompressed_limit=None):
self.c = c
self.poll_interval = 300
self.workernames = workernames
@@ -58,6 +58,7 @@
self.artifact_link_base = normalize_base_url(artifact_link_base)
self.artifact_uncompressed_limit = artifact_uncompressed_limit or default_artifact_uncompressed_limit
self.artifact_compressed_limit = artifact_compressed_limit or default_artifact_compressed_limit
+ self.pages_uncompressed_limit = pages_uncompressed_limit or self.artifact_uncompressed_limit
def configure_factory_and_builder(self):
self.add_lilybuild_builder()
@@ -160,6 +161,7 @@
artifact_compressed_limit=None,
artifact_uncompressed_limit=None,
artifact_latest_branch_map=None,
+ pages_uncompressed_limit=None,
):
if not alternative_urls:
alternative_urls = []
@@ -175,6 +177,7 @@
'artifact_compressed_limit': artifact_compressed_limit or self.artifact_compressed_limit,
'artifact_uncompressed_limit': artifact_uncompressed_limit or self.artifact_uncompressed_limit,
'artifact_latest_branch_map': artifact_latest_branch_map or {},
+ 'pages_uncompressed_limit': pages_uncompressed_limit or self.pages_uncompressed_limit,
}
self.record_url(repo_id, repo_url)
for u in alternative_urls:
diff --git a/lilybuild/lilybuild/pages.py b/lilybuild/lilybuild/pages.py
--- a/lilybuild/lilybuild/pages.py
+++ b/lilybuild/lilybuild/pages.py
@@ -71,17 +71,22 @@
self.total_bytes_extracted += filtered_member.size
return filtered_member
-def extract_archive(dir_name, archive_file):
+def extract_archive(dir_name, archive_file, limit_bytes=None):
+ if limit_bytes is None:
+ limit_bytes = 100 * 1024 * 1024
with tarfile.open(archive_file) as tf:
tf.errorlevel = 1
- tf.extractall(dir_name, filter=TarPagesFilter())
+ tf.extractall(dir_name, filter=TarPagesFilter(limit_bytes=limit_bytes))
-def deploy_pages(dir_name, archive_file):
+def deploy_pages(dir_name, archive_file, max_size=None):
target = init_deployment(dir_name)
full_target = os.path.join(dir_name, target)
empty_directory(full_target)
- extract_archive(full_target, archive_file)
+ extract_archive(full_target, archive_file, limit_bytes=max_size)
switch_symlink(dir_name, target)
if __name__ == '__main__':
- deploy_pages(sys.argv[1], sys.argv[2])
+ max_size = None
+ if len(sys.argv) >= 4:
+ max_size = int(sys.argv[3])
+ deploy_pages(sys.argv[1], sys.argv[2], max_size=max_size)
diff --git a/lilybuild/lilybuild/tests/pages_test.py b/lilybuild/lilybuild/tests/pages_test.py
--- a/lilybuild/lilybuild/tests/pages_test.py
+++ b/lilybuild/lilybuild/tests/pages_test.py
@@ -80,7 +80,7 @@
with tempfile.TemporaryDirectory() as root_dir:
archive_file = make_artifact_archive(root_dir)
dir_name = os.path.join(root_dir, 'deployment')
- extract_archive(dir_name, archive_file)
+ extract_archive(dir_name, archive_file, limit_bytes=100*1024*1024)
self.assertTrue(is_dir(os.path.join(dir_name, 'public')))
self.assertTrue(os.path.exists(os.path.join(dir_name, 'public', 'a')))
self.assertFalse(os.path.exists(os.path.join(dir_name, 'other')))
@@ -90,7 +90,7 @@
archive_file = make_bad_artifact_archive(root_dir)
dir_name = os.path.join(root_dir, 'deployment')
with self.assertRaises(tarfile.FilterError):
- extract_archive(dir_name, archive_file)
+ extract_archive(dir_name, archive_file, limit_bytes=100*1024*1024)
def test_deploy_pages(self):
with tempfile.TemporaryDirectory() as root_dir:
@@ -98,14 +98,14 @@
bad_archive = make_bad_artifact_archive(os.path.join(root_dir, 'bad'))
archive_file2 = make_artifact_archive2(os.path.join(root_dir, 'a2'))
dir_name = os.path.join(root_dir, 'deployment')
- deploy_pages(dir_name, archive_file)
+ deploy_pages(dir_name, archive_file, max_size=100*1024*1024)
self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), FIRST)
self.assertTrue(os.path.exists(os.path.join(dir_name, FIRST, 'public', 'a')))
with open(os.path.join(dir_name, FIRST, 'public', 'a')) as f:
self.assertEqual(f.read(), 'test\n')
# deploy the same thing again, it should go to SECOND
- deploy_pages(dir_name, archive_file)
+ deploy_pages(dir_name, archive_file, max_size=100*1024*1024)
self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)
self.assertTrue(os.path.exists(os.path.join(dir_name, SECOND, 'public', 'a')))
with open(os.path.join(dir_name, SECOND, 'public', 'a')) as f:
@@ -113,11 +113,11 @@
# try to deploy the bad archive, it should not update the link
with self.assertRaises(tarfile.FilterError):
- deploy_pages(dir_name, bad_archive)
+ deploy_pages(dir_name, bad_archive, max_size=100*1024*1024)
self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)
# deploy something else, verify it cleans up all old files
- deploy_pages(dir_name, archive_file2)
+ deploy_pages(dir_name, archive_file2, max_size=100*1024*1024)
self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), FIRST)
self.assertTrue(os.path.exists(os.path.join(dir_name, FIRST, 'public', 'b')))
self.assertFalse(os.path.exists(os.path.join(dir_name, FIRST, 'public', 'a')))

File Metadata

Mime Type
text/plain
Expires
Thu, May 28, 12:34 AM (12 h, 10 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1508998
Default Alt Text
D305.1779953669.diff (6 KB)

Event Timeline