Page MenuHomePhorge
Files F2577277

D213.1750680605.diff
No OneTemporary
Actions

Size
8 KB
Referenced Files
None
Subscribers
None

D213.1750680605.diff
View Options

diff --git a/lilybuild/lilybuild/pages.py b/lilybuild/lilybuild/pages.py
new file mode 100644
--- /dev/null
+++ b/lilybuild/lilybuild/pages.py
@@ -0,0 +1,80 @@
+
+import os
+import subprocess
+import tarfile
+import shutil
+import sys
+
+'''
+This module implements blue-green deployment for pages.
+'''
+
+FIRST = '0'
+SECOND = '1'
+CURRENT = 'cur'
+
+def init_deployment(dir_name):
+ '''
+ Init the deployment at `dir_name` and return which directory we should
+ write into for the next deployment.
+ '''
+ os.makedirs(os.path.join(dir_name, FIRST), exist_ok=True)
+ os.makedirs(os.path.join(dir_name, SECOND), exist_ok=True)
+ link_file = os.path.join(dir_name, CURRENT)
+ try:
+ current = os.readlink(link_file)
+ except FileNotFoundError as e:
+ os.symlink(SECOND, link_file, target_is_directory=True)
+ current = SECOND
+ return FIRST if current == SECOND else SECOND
+
+def switch_symlink(dir_name, target):
+ subprocess.run([
+ 'ln',
+ '-sfvn',
+ '--',
+ target,
+ os.path.join(dir_name, CURRENT),
+ ], check=True)
+
+def empty_directory(dir_name):
+ shutil.rmtree(dir_name)
+ os.makedirs(dir_name, exist_ok=True)
+
+class TarPagesFilter:
+ public_dir = 'public'
+ def __init__(
+ self,
+ limit_bytes=100*1024*1024 # 100 MiB
+ ):
+ self.total_bytes_extracted = 0
+ self.limit_bytes = limit_bytes
+
+ def __call__(self, member, path):
+ filtered_member = tarfile.data_filter(member, path)
+ if not filtered_member:
+ return None
+ if self.total_bytes_extracted + filtered_member.size > self.limit_bytes:
+ self.total_bytes_extracted = self.limit_bytes + 1
+ raise RuntimeError('Limit exceeded')
+ name = os.path.normpath(filtered_member.name)
+ if not (name == self.public_dir or name.startswith(self.public_dir + '/')):
+ return None
+
+ self.total_bytes_extracted += filtered_member.size
+ return filtered_member
+
+def extract_archive(dir_name, archive_file):
+ with tarfile.open(archive_file) as tf:
+ tf.errorlevel = 1
+ tf.extractall(dir_name, filter=TarPagesFilter())
+
+def deploy_pages(dir_name, archive_file):
+ target = init_deployment(dir_name)
+ full_target = os.path.join(dir_name, target)
+ empty_directory(full_target)
+ extract_archive(full_target, archive_file)
+ switch_symlink(dir_name, target)
+
+if __name__ == '__main__':
+ deploy_pages(sys.argv[1], sys.argv[2])
diff --git a/lilybuild/lilybuild/tests/pages_test.py b/lilybuild/lilybuild/tests/pages_test.py
new file mode 100644
--- /dev/null
+++ b/lilybuild/lilybuild/tests/pages_test.py
@@ -0,0 +1,122 @@
+
+import unittest
+import tempfile
+import tarfile
+import os
+import stat
+import subprocess
+from lilybuild.pages import (
+ FIRST, SECOND, CURRENT, init_deployment,
+ switch_symlink, extract_archive, deploy_pages,
+)
+
+def is_dir(filename):
+ return stat.S_ISDIR(os.lstat(filename).st_mode)
+
+def make_artifact_archive(root_dir):
+ os.makedirs(os.path.join(root_dir, 'public'))
+ with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
+ print('test', file=f)
+ os.makedirs(os.path.join(root_dir, 'other'))
+ with open(os.path.join(root_dir, 'other', 'a'), 'w') as f:
+ print('should not be there', file=f)
+ archive = os.path.join(root_dir, 'artifacts.tar')
+ with tarfile.open(archive, 'w') as f:
+ f.add(os.path.join(root_dir, 'public'), 'public')
+ f.add(os.path.join(root_dir, 'other'), 'other')
+ return archive
+
+def make_artifact_archive2(root_dir):
+ os.makedirs(os.path.join(root_dir, 'public'))
+ with open(os.path.join(root_dir, 'public', 'b'), 'w') as f:
+ print('test b', file=f)
+ archive = os.path.join(root_dir, 'artifacts.tar')
+ with tarfile.open(archive, 'w') as f:
+ f.add(os.path.join(root_dir, 'public'), 'public')
+ return archive
+
+def make_bad_artifact_archive(root_dir):
+ os.makedirs(os.path.join(root_dir, 'public'))
+ with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
+ print('test', file=f)
+ os.makedirs(os.path.join(root_dir, 'other'))
+ with open(os.path.join(root_dir, 'other', 'a'), 'w') as f:
+ print('should not be there', file=f)
+ archive = os.path.join(root_dir, 'artifacts.tar')
+ with tarfile.open(archive, 'w') as f:
+ f.add(os.path.join(root_dir, 'public'), 'public')
+ f.add(os.path.join(root_dir, 'other'), '../../../other')
+ return archive
+
+class PagesTest(unittest.TestCase):
+ def test_init_deployment(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ res = init_deployment(dir_name)
+ self.assertEqual(res, FIRST)
+ self.assertTrue(is_dir(os.path.join(dir_name, FIRST)))
+ self.assertTrue(is_dir(os.path.join(dir_name, SECOND)))
+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)
+
+ res = init_deployment(dir_name)
+ self.assertEqual(res, FIRST)
+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)
+
+ def test_switch_symlink(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ res = init_deployment(dir_name)
+ switch_symlink(dir_name, res)
+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), FIRST)
+
+ def test_extract_archive(self):
+ with tempfile.TemporaryDirectory() as root_dir:
+ archive_file = make_artifact_archive(root_dir)
+ dir_name = os.path.join(root_dir, 'deployment')
+ extract_archive(dir_name, archive_file)
+ self.assertTrue(is_dir(os.path.join(dir_name, 'public')))
+ self.assertTrue(os.path.exists(os.path.join(dir_name, 'public', 'a')))
+ self.assertFalse(os.path.exists(os.path.join(dir_name, 'other')))
+
+ def test_extract_bad_archive(self):
+ with tempfile.TemporaryDirectory() as root_dir:
+ archive_file = make_bad_artifact_archive(root_dir)
+ dir_name = os.path.join(root_dir, 'deployment')
+ with self.assertRaises(tarfile.FilterError):
+ extract_archive(dir_name, archive_file)
+
+ def test_deploy_pages(self):
+ with tempfile.TemporaryDirectory() as root_dir:
+ archive_file = make_artifact_archive(os.path.join(root_dir, 'a1'))
+ bad_archive = make_bad_artifact_archive(os.path.join(root_dir, 'bad'))
+ archive_file2 = make_artifact_archive2(os.path.join(root_dir, 'a2'))
+ dir_name = os.path.join(root_dir, 'deployment')
+ deploy_pages(dir_name, archive_file)
+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), FIRST)
+ self.assertTrue(os.path.exists(os.path.join(dir_name, FIRST, 'public', 'a')))
+ with open(os.path.join(dir_name, FIRST, 'public', 'a')) as f:
+ self.assertEqual(f.read(), 'test\n')
+
+ # deploy the same thing again, it should go to SECOND
+ deploy_pages(dir_name, archive_file)
+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)
+ self.assertTrue(os.path.exists(os.path.join(dir_name, SECOND, 'public', 'a')))
+ with open(os.path.join(dir_name, SECOND, 'public', 'a')) as f:
+ self.assertEqual(f.read(), 'test\n')
+
+ # try to deploy the bad archive, it should not update the link
+ with self.assertRaises(tarfile.FilterError):
+ deploy_pages(dir_name, bad_archive)
+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)
+
+ # deploy something else, verify it cleans up all old files
+ deploy_pages(dir_name, archive_file2)
+ self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), FIRST)
+ self.assertTrue(os.path.exists(os.path.join(dir_name, FIRST, 'public', 'b')))
+ self.assertFalse(os.path.exists(os.path.join(dir_name, FIRST, 'public', 'a')))
+ self.assertTrue(os.path.exists(os.path.join(dir_name, SECOND, 'public', 'a')))
+ self.assertFalse(os.path.exists(os.path.join(dir_name, SECOND, 'public', 'b')))
+ with open(os.path.join(dir_name, FIRST, 'public', 'b')) as f:
+ self.assertEqual(f.read(), 'test b\n')
+
+
+if __name__ == '__main__':
+ unittest.main()

File Metadata

Mime Type
text/plain
Expires
Mon, Jun 23, 5:10 AM (7 h, 12 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
234688
Default Alt Text
D213.1750680605.diff (8 KB)

Event Timeline