[lxc-devel] [pylxd/master] Added functionality to recursively pull a directory from the instance…
anneborcherding on Github
lxc-bot at linuxcontainers.org
Wed Jul 15 08:49:18 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 502 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200715/0d48d45c/attachment.bin>
-------------- next part --------------
From 8e43bf4572f5bc3ed3cdabe0a60a81b08b8db4a6 Mon Sep 17 00:00:00 2001
From: Anne Borcherding <anne.borcherding at iosb.fraunhofer.de>
Date: Wed, 15 Jul 2020 10:46:24 +0200
Subject: [PATCH] Added functionality to recursively pull a directory from the
instance and to push an empty directory to the instance.
Co-authored-by: weichweich <14820950+weichweich at users.noreply.github.com>
Signed-off-by: Anne Borcherding <anne.borcherding at iosb.fraunhofer.de>
---
CONTRIBUTORS.rst | 2 +
doc/source/containers.rst | 3 +
pylxd/models/instance.py | 61 ++++++++++++++
pylxd/tests/models/test_instance.py | 125 ++++++++++++++++++++++++++--
4 files changed, 186 insertions(+), 5 deletions(-)
diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst
index 3c73dbcd..041c6646 100644
--- a/CONTRIBUTORS.rst
+++ b/CONTRIBUTORS.rst
@@ -36,5 +36,7 @@ These are the contributors to pylxd according to the Github repository.
mrtc0 Kohei Morita
gabrik Gabriele Baldoni
felix-engelmann Felix Engelmann
+ weichweich ???
+ anneborcherding Anne Borcherding
=============== ==================================
diff --git a/doc/source/containers.rst b/doc/source/containers.rst
index 3ab99fc3..fe13a271 100644
--- a/doc/source/containers.rst
+++ b/doc/source/containers.rst
@@ -237,7 +237,10 @@ Containers also have a `files` manager for getting and putting files on the
container. The following methods are available on the `files` manager:
- `put` - push a file into the container.
+ - `put_dir` - push an empty directory to the container.
+ - `recursive_put` - recursively push a directory to the container.
- `get` - get a file from the container.
+ - `recursive_get` - recursively pull a directory from the container.
- `delete_available` - If the `file_delete` extension is available on the lxc
host, then this method returns `True` and the `delete` method is available.
- `delete` - delete a file on the container.
diff --git a/pylxd/models/instance.py b/pylxd/models/instance.py
index c45ec431..3c0b2a2e 100644
--- a/pylxd/models/instance.py
+++ b/pylxd/models/instance.py
@@ -12,12 +12,14 @@
# License for the specific language governing permissions and limitations
# under the License.
import collections
+import json
import os
import stat
import time
import six
from six.moves.urllib import parse
+
try:
from ws4py.client import WebSocketBaseClient
from ws4py.manager import WebSocketManager
@@ -123,6 +125,33 @@ def put(self, filepath, data, mode=None, uid=None, gid=None):
return
raise LXDAPIException(response)
+ def put_dir(self, path, mode=None, uid=None, gid=None):
+ """Push an empty directory to the container.
+ This pushes an empty directory to the containers file system
+ named by the `filepath`.
+ :param path: The path in the container to to store the data in.
+ :type path: str
+ :param mode: The unit mode to store the file with. The default of
+ None stores the file with the current mask of 0700, which is
+ the lxd default.
+ :type mode: Union[oct, int, str]
+ :param uid: The uid to use inside the container. Default of None
+ results in 0 (root).
+ :type uid: int
+ :param gid: The gid to use inside the container. Default of None
+ results in 0 (root).
+ :type gid: int
+ :raises: LXDAPIException if something goes wrong
+ """
+ headers = self._resolve_headers(mode=mode, uid=uid, gid=gid)
+ headers['X-LXD-type'] = 'directory'
+ response = self._endpoint.post(
+ params={'path': path},
+ headers=headers or None)
+ if response.status_code == 200:
+ return
+ raise LXDAPIException(response)
+
@staticmethod
def _resolve_headers(headers=None, mode=None, uid=None, gid=None):
if headers is None:
@@ -224,6 +253,38 @@ def recursive_put(self, src, dst, mode=None, uid=None, gid=None):
if response.status_code != 200:
raise LXDAPIException(response)
+ def recursive_get(self, remote_path, local_path):
+ """Recursively pulls a directory from the container.
+ Pulls the directory named `remote_path` from the container and
+ creates a local folder named `local_path` with the
+ content of `remote_path`.
+ If `remote_path` is a file, it will be copied to `local_path`.
+ :param remote_path: The directory path on the container.
+ :type remote_path: str
+ :param local_path: The path at which the directory will be stored.
+ :type local_path: str
+ :return:
+ :raises: LXDAPIException if an error occurs
+ """
+ response = self._endpoint.get(
+ params={'path': remote_path}, is_api=False)
+
+ print(response)
+ if "X-LXD-type" in response.headers:
+ print("1")
+ if response.headers["X-LXD-type"] == "directory":
+ print("2")
+ os.mkdir(local_path)
+ print(response.content)
+ content = json.loads(response.content)
+ if "metadata" in content and content["metadata"]:
+ for file in content["metadata"]:
+ self.recursive_get(os.path.join(remote_path, file),
+ os.path.join(local_path, file))
+ elif response.headers["X-LXD-type"] == "file":
+ with open(local_path, "wb") as f:
+ f.write(response.content)
+
@classmethod
def exists(cls, client, name):
"""Determine whether a instance exists."""
diff --git a/pylxd/tests/models/test_instance.py b/pylxd/tests/models/test_instance.py
index 3c6c165b..e44f0fe6 100644
--- a/pylxd/tests/models/test_instance.py
+++ b/pylxd/tests/models/test_instance.py
@@ -5,6 +5,7 @@
import tempfile
import mock
+import requests
from six.moves.urllib.parse import quote as url_quote
@@ -31,12 +32,14 @@ def test_get(self):
def test_get_not_found(self):
"""LXDAPIException is raised when the instance doesn't exist."""
+
def not_found(request, context):
context.status_code = 404
return json.dumps({
'type': 'error',
'error': 'Not found',
'error_code': 404})
+
self.add_rule({
'text': not_found,
'method': 'GET',
@@ -51,12 +54,14 @@ def not_found(request, context):
def test_get_error(self):
"""LXDAPIException is raised when the LXD API errors."""
+
def not_found(request, context):
context.status_code = 500
return json.dumps({
'type': 'error',
'error': 'Not found',
'error_code': 500})
+
self.add_rule({
'text': not_found,
'method': 'GET',
@@ -96,12 +101,14 @@ def test_exists(self):
def test_not_exists(self):
"""A instance exists."""
+
def not_found(request, context):
context.status_code = 404
return json.dumps({
'type': 'error',
'error': 'Not found',
'error_code': 404})
+
self.add_rule({
'text': not_found,
'method': 'GET',
@@ -123,12 +130,14 @@ def test_fetch(self):
def test_fetch_not_found(self):
"""LXDAPIException is raised on a 404 for updating instance."""
+
def not_found(request, context):
context.status_code = 404
return json.dumps({
'type': 'error',
'error': 'Not found',
'error_code': 404})
+
self.add_rule({
'text': not_found,
'method': 'GET',
@@ -142,12 +151,14 @@ def not_found(request, context):
def test_fetch_error(self):
"""LXDAPIException is raised on error."""
+
def not_found(request, context):
context.status_code = 500
return json.dumps({
'type': 'error',
'error': 'An bad error',
'error_code': 500})
+
self.add_rule({
'text': not_found,
'method': 'GET',
@@ -241,6 +252,7 @@ def test_execute_no_ws4py(self):
def cleanup():
instance._ws4py_installed = old_installed
+
self.addCleanup(cleanup)
an_instance = models.Instance(
@@ -395,9 +407,9 @@ def test_publish(self):
'metadata': {
'fingerprint': ('e3b0c44298fc1c149afbf4c8996fb92427'
'ae41e4649b934ca495991b7852b855')
- }
}
- }),
+ }
+ }),
'method': 'GET',
'url': r'^http://pylxd.test/1.0/operations/operation-abc$',
})
@@ -523,12 +535,14 @@ def test_delete(self):
def test_delete_failure(self):
"""If the response indicates delete failure, raise an exception."""
+
def not_found(request, context):
context.status_code = 404
return json.dumps({
'type': 'error',
'error': 'Not found',
'error_code': 404})
+
self.add_rule({
'text': not_found,
'method': 'DELETE',
@@ -552,9 +566,9 @@ def test_publish(self):
'metadata': {
'fingerprint': ('e3b0c44298fc1c149afbf4c8996fb92427'
'ae41e4649b934ca495991b7852b855')
- }
}
- }),
+ }
+ }),
'method': 'GET',
'url': r'^http://pylxd.test/1.0/operations/operation-abc$',
})
@@ -597,7 +611,7 @@ def test_put_delete(self):
'metadata': {'auth': 'trusted',
'environment': {
'certificate': 'an-pem-cert',
- },
+ },
'api_extensions': ['file_delete']
}}),
'method': 'GET',
@@ -665,6 +679,38 @@ def capture(request, context):
with self.assertRaises(ValueError):
self.instance.files.put('/tmp/putted', data, mode=object)
+ def test_put_dir(self):
+ """Tests pushing an empty directory"""
+ _capture = {}
+
+ def capture(request, context):
+ _capture['headers'] = getattr(request._request, 'headers')
+ context.status_code = 200
+
+ self.add_rule({
+ 'text': capture,
+ 'method': 'POST',
+ 'url': (r'^http://pylxd.test/1.0/instances/an-instance/files'
+ r'\?path=%2Ftmp%2Fputted$'),
+ })
+
+ self.instance.files.put_dir('/tmp/putted', mode=0o123, uid=1, gid=2)
+ headers = _capture['headers']
+ self.assertEqual(headers['X-LXD-type'], 'directory')
+ self.assertEqual(headers['X-LXD-mode'], '0123')
+ self.assertEqual(headers['X-LXD-uid'], '1')
+ self.assertEqual(headers['X-LXD-gid'], '2')
+ # check that assertion is raised
+ with self.assertRaises(ValueError):
+ self.instance.files.put_dir('/tmp/putted', mode=object)
+
+ response = mock.Mock()
+ response.status_code = 404
+
+ with mock.patch('pylxd.client._APINode.post', response):
+ with self.assertRaises(exceptions.LXDAPIException):
+ self.instance.files.put_dir('/tmp/putted')
+
def test_recursive_put(self):
@contextlib.contextmanager
@@ -738,10 +784,77 @@ def test_get(self):
self.assertEqual(b'This is a getted file', data)
+ def test_recursive_get(self):
+ """A folder is retrieved recursively from the instance"""
+
+ @contextlib.contextmanager
+ def tempdir(prefix='tmp'):
+ tmpdir = tempfile.mkdtemp(prefix=prefix)
+ try:
+ yield tmpdir
+ finally:
+ shutil.rmtree(tmpdir)
+
+ def create_file(_dir, name, content):
+ path = os.path.join(_dir, name)
+ actual_dir = os.path.dirname(path)
+ if not os.path.exists(actual_dir):
+ os.makedirs(actual_dir)
+ with open(path, 'w') as f:
+ f.write(content)
+
+ _captures = []
+
+ def capture(request, context):
+ _captures.append({
+ 'headers': getattr(request._request, 'headers'),
+ 'body': request._request.body,
+ })
+ context.status_code = 200
+
+ response = requests.models.Response()
+ response.status_code = 200
+ response.headers["X-LXD-type"] = "directory"
+ response._content = json.dumps({'metadata': ['file1', 'file2']})
+
+ response1 = requests.models.Response()
+ response1.status_code = 200
+ response1.headers["X-LXD-type"] = "file"
+ response1._content = "This is file1"
+
+ response2 = requests.models.Response()
+ response2.status_code = 200
+ response2.headers["X-LXD-type"] = "file"
+ response2._content = "This is file2"
+
+ return_values = [response, response1, response2]
+
+ with mock.patch('pylxd.client._APINode.get') as get_mocked:
+ get_mocked.side_effect = return_values
+ with mock.patch('os.mkdir') as mkdir_mocked:
+ # distinction needed for the code to work with python2.7 and 3
+ try:
+ with mock.patch('__builtin__.open') as open_mocked:
+ self.instance.files\
+ .recursive_get('/tmp/getted', '/tmp')
+ assert (mkdir_mocked.call_count == 1)
+ assert(open_mocked.call_count == 2)
+ except ModuleNotFoundError:
+ try:
+ with mock.patch('builtins.open') as open_mocked:
+ self.instance.files\
+ .recursive_get('/tmp/getted', '/tmp')
+ assert (mkdir_mocked.call_count == 1)
+ assert (open_mocked.call_count == 2)
+ except ModuleNotFoundError as e:
+ raise e
+
def test_get_not_found(self):
"""LXDAPIException is raised on bogus filenames."""
+
def not_found(request, context):
context.status_code = 500
+
rule = {
'text': not_found,
'method': 'GET',
@@ -756,8 +869,10 @@ def not_found(request, context):
def test_get_error(self):
"""LXDAPIException is raised on error."""
+
def not_found(request, context):
context.status_code = 503
+
rule = {
'text': not_found,
'method': 'GET',
More information about the lxc-devel
mailing list