diff --git a/declearn/communication3/server/__init__.py b/declearn/communication3/server/__init__.py
index 70c7fe3a8d303a1bb0427750ffed4f1c6177acc6..4097c1cff0bf904d72aa96cb26b7c580e7d3cea5 100644
--- a/declearn/communication3/server/__init__.py
+++ b/declearn/communication3/server/__init__.py
@@ -17,4 +17,5 @@
"""DecLearn v3 experimental network server endpoint code."""
+from ._files import FileAuthError, FileExpiredError, FilesManager
from ._peers import PeerPolicy, PeersManager
diff --git a/declearn/communication3/server/_files.py b/declearn/communication3/server/_files.py
new file mode 100644
index 0000000000000000000000000000000000000000..309f1d401f775b8ae67582242e376abe5fd93d70
--- /dev/null
+++ b/declearn/communication3/server/_files.py
@@ -0,0 +1,377 @@
+# coding: utf-8
+
+# Copyright 2023 Inria (Institut National de Recherche en Informatique
+# et Automatique)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Component to manage binary files storage for a communication server."""
+
+import datetime
+import os
+import secrets
+from typing import Dict, Iterator, Optional, Tuple, Union
+
+from declearn.communication3.server._utils import create_secret_token
+
+__all__ = [
+ "FilesManager",
+ "FileAuthError",
+ "FileExpiredError",
+]
+
+
+class FileAuthError(Exception):
+ """Custom exception denoting wrong authentication to access a SavedFile."""
+
+
+class FileExpiredError(Exception):
+ """Custom exception denoting that a SavedFile has expired."""
+
+
+class SavedFile:
+ """Class to handle a saved binary file on a communication server.
+
+ This class is merely a backend tool, to be used by `FilesManager`.
+ It may be refactored or replaced in the future, notably using some
+ database to store information on files' location, secret tokens,
+ metadata and access logs.
+ """
+
+ # backend class with private attributes;
+ # pylint: disable=too-many-instance-attributes
+
+ def __init__(
+ self,
+ path: str,
+ access_token: str,
+ delete_token: str,
+ max_accesses: Optional[int] = None,
+ max_keeptime: Optional[datetime.timedelta] = None,
+ ) -> None:
+ """Instantiate the SavedFile handler.
+
+ Parameters
+ ----------
+ path:
+ Path to the file being watched and managed.
+ access_token:
+ Secret token required to open the file and read its contents.
+ delete_token:
+ Secret token required to perform non-automated file deletion.
+ max_accesses:
+ Optional maximum number of accesses, beyond which to disable
+ any new file access, and enable automatic deletion.
+ max_keeptime:
+ Optional maximum storage duration, beyond which to disable
+ any new file access, and enable automatic deletion.
+ """
+ # arguments serve modularity; pylint: disable=too-many-arguments
+ self._path = path
+ self._access_token = access_token
+ self._delete_token = delete_token
+ self._max_accesses = max_accesses
+ self._num_accesses = 0
+ self._srt_datetime = datetime.datetime.now()
+ self._max_keeptime = max_keeptime
+ self._n_open_reads = 0
+ self._delete_next = False
+
+ def access(
+ self,
+ token: str,
+ ) -> Iterator[bytes]:
+ """Access the contents from the file.
+
+ Parameters
+ ----------
+ token:
+ Secret token enabling to access this file.
+
+ Yields
+ ------
+ rows:
+ Rows of bytes content, iteratively read from the file.
+
+ Raises
+ ------
+ FileAuthError
+ If the provided token does not grant access rights.
+
+ Notes
+ -----
+ - File expiration and access token verification are performed
+ on call, to prevent race conditions from streaming from the
+ returned generator.
+ - File opening, on the other hand, is delegated to the first
+ iteration on the returned generator. Similarly, reading is
+ performed iteratively. Fail-safes should prevent the file
+ from being deleted while being accessed.
+ - Upon yielding the final chunk of content, if the file has
+ expired and no other access is open, it is automatically
+ deleted from disk.
+ """
+ iterator = self._access(token)
+ next(iterator) # trigger access validation code
+ return iterator
+
+ def _access(
+ self,
+ token: str,
+ ) -> Iterator[bytes]:
+ """Backend code for the `access` method.
+
+ This method is merely here to enable triggering file expiration
+ and token verification right on `access` call, while preserving
+ the iterative nature of the file's contents streaming.
+ """
+ # Verify that the access is authorized.
+ if self.expired:
+ raise FileExpiredError("Cannot access expired file.")
+ if not secrets.compare_digest(token, self._access_token):
+ raise FileAuthError("Incorrect access token.")
+ # Increment counters on total number of accesses and current opens.
+ self._num_accesses += 1
+ self._n_open_reads += 1
+ yield b"" # enable triggering the code above right away in `access`
+ # Yield the file's contents; auto-delete at close time.
+ with open(self._path, "rb") as file:
+ yield from file
+ self._n_open_reads -= 1
+ self.auto_delete()
+
+ def delete(
+ self,
+ token: str,
+ ) -> None:
+ """Delete this file from disk.
+
+ Parameters
+ ----------
+ token:
+ Secret token enabling to remove this file.
+
+ Raises
+ ------
+ FileAuthError
+ If the provided token does not grant deletion rights.
+
+ Notes
+ -----
+ - If the file is currently being accessed, delay its deletion
+ to the end of these accesses, and refuse any new access.
+ - If the file has already been removed from disk, still verify
+ the input token, and silently pass or raise based on that.
+ In other words, do not let the caller know that the file has
+ already been deleted.
+ """
+ if not secrets.compare_digest(token, self._delete_token):
+ raise FileAuthError("Incorrect deletion token.")
+ if self._n_open_reads:
+ self._delete_next = True
+ elif os.path.isfile(self._path):
+ os.remove(self._path)
+
+ @property
+ def expired(
+ self,
+ ) -> bool:
+ """Whether this file has expired - as a property evaluated on call."""
+ if self._delete_next or not os.path.isfile(self._path):
+ return True
+ if self._max_accesses is not None:
+ if self._num_accesses >= self._max_accesses:
+ return True
+ if self._max_keeptime is not None:
+ keeptime = datetime.datetime.now() - self._srt_datetime
+ if keeptime >= self._max_keeptime:
+ return True
+ return False
+
+ def auto_delete(
+ self,
+ ) -> None:
+ """Automatically delete this file from disk if it has expired."""
+ if self.expired:
+ self.delete(self._delete_token)
+
+
+class FilesManager:
+ """Component to manage binary files storage for a communication server."""
+
+ def __init__(
+ self,
+ folder: str,
+ ) -> None:
+ """Instantiate the stored files manager.
+
+ Parameters
+ ----------
+ folder:
+ Path to the root folder where to create stored files.
+ """
+ os.makedirs(folder, exist_ok=True)
+ self.folder = folder
+ self._files = {} # type: Dict[str, SavedFile]
+
+ def create_file(
+ self,
+ buffer: Union[bytes, bytearray],
+ prefix: Optional[str] = None,
+ max_accesses: Optional[int] = None,
+ max_keeptime: Optional[datetime.timedelta] = None,
+ ) -> Tuple[str, str, str]:
+ """Add a file to the manager.
+
+ Parameters
+ ----------
+ buffer:
+ Byte content (or stream) of the file to create and manage.
+ prefix:
+ Optional prefix to used in the registered name of the file.
+ max_accesses:
+ Optional maximum number of accesses to the file, leading to
+ its automatic expiration and removal.
+ max_keeptime:
+ Optional maximum duration of storage of the file, leading to
+ its automatic expiration and removal.
+
+ Returns
+ -------
+ name:
+ Registered name of the saved file.
+ access_token:
+ Secret token enabling to access the saved file.
+ delete_token:
+ Secret token enabling to remove the saved file.
+ """
+ # Select a public name that is not already in use.
+ while (
+ name := f"{prefix or 'file'}_{secrets.token_urlsafe()}"
+ ) in self._files:
+ pass
+ # Select a secret path that is not already in use.
+ while os.path.isfile(
+ path := os.path.join(self.folder, str(secrets.token_hex()))
+ ):
+ pass
+ # Write down the file.
+ with open(path, "wb") as file:
+ file.write(buffer)
+ # Create random secret tokens.
+ access_token = create_secret_token()
+ delete_token = create_secret_token()
+ # Store a reference to the managed file, and return access information.
+ self._files[name] = SavedFile(
+ path=path,
+ access_token=access_token,
+ delete_token=delete_token,
+ max_accesses=max_accesses,
+ max_keeptime=max_keeptime,
+ )
+ return name, access_token, delete_token
+
+ def access_file(
+ self,
+ name: str,
+ token: str,
+ ) -> Iterator[bytes]:
+ """Access a saved file.
+
+ Parameters
+ ----------
+ name:
+ Registered name of the file to access.
+ token:
+ Authentication token to access the file.
+
+ Yields
+ ------
+ stream:
+ Bytes content from the file, as a generator streaming
+ the file one row as a time.
+
+ Raises
+ ------
+ FileAuthError
+ If the authentication token is not the proper one.
+ FileExpiredError
+ If the file has expired and can no longer be accessed.
+ FileNotFoundError
+ If the file name does not match a managed resource.
+
+ Notes
+ -----
+ - File expiration and access token verification are performed
+ on call.
+ - File opening and reading are performed iteratively, based on
+ actions performed on the returned generator.
+ - Until the entire file has been read, it will not be removed
+ from disk, even if its deletion is asked. In the latter case,
+ deletion will be delayed until all pre-existing accesses are
+ closed.
+ """
+ if name not in self._files:
+ raise FileNotFoundError(f"No file resource with name {name}.")
+ return self._files[name].access(token)
+
+ def delete_file(
+ self,
+ name: str,
+ token: str,
+ deref: bool = False,
+ ) -> None:
+ """Delete a saved file.
+
+ Parameters
+ ----------
+ name:
+ Registered name of the file to delete.
+ token:
+ Authentication token to delete the file.
+ deref:
+ Whether to de-reference the file, so that new access attempts
+ will raise `FileNotFoundError` rather than `FileExpiredError`.
+
+ Raises
+ ------
+ FileAuthError
+ If the authentication token is not the proper one.
+ FileNotFoundError
+ If the file name does not match a managed resource.
+
+ Notes
+ -----
+ - If the file has already been deleted but is still referenced by
+ this manager, this function will pass or raise all the same.
+ - If the file is currently being accessed, its deletion will be
+ delayed to the termination of these accesses, and no new access
+ will be allowed.
+ """
+ if name not in self._files:
+ raise FileNotFoundError(f"No file resource with name {name}.")
+ self._files[name].delete(token)
+ if deref:
+ self._files.pop(name)
+
+ def remove_expired_references(
+ self,
+ ) -> None:
+ """Remove references to managed files that have expired.
+
+ Delete expired files from disk upon being de-referenced.
+ """
+ expired = [name for name, fobj in self._files.items() if fobj.expired]
+ for name in expired:
+ fobj = self._files.pop(name)
+ fobj.auto_delete()
diff --git a/declearn/communication3/server/_peers.py b/declearn/communication3/server/_peers.py
index 73eda459e9bafb62cc647b0ef2b84f00960d9698..f2054de63c1459769bf6ed472bdab2ecd481913d 100644
--- a/declearn/communication3/server/_peers.py
+++ b/declearn/communication3/server/_peers.py
@@ -20,9 +20,9 @@
import copy
import dataclasses
import secrets
-
from typing import Dict, Optional, Set, Tuple
+from declearn.communication3.server._utils import create_secret_token
__all__ = [
"PeersManager",
@@ -30,14 +30,6 @@ __all__ = [
]
-DEFAULT_TOKEN_NBYTES = 64 # Default number of bytes in user auth tokens.
-
-
-def create_secret_token() -> str:
- """Create a random secret string token."""
- return secrets.token_hex(nbytes=DEFAULT_TOKEN_NBYTES)
-
-
MOCK_TOKEN = create_secret_token()
diff --git a/declearn/communication3/server/_utils.py b/declearn/communication3/server/_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..8f58750dff91a362d36fab1e3c27f2c19bbba959
--- /dev/null
+++ b/declearn/communication3/server/_utils.py
@@ -0,0 +1,32 @@
+# coding: utf-8
+
+# Copyright 2023 Inria (Institut National de Recherche en Informatique
+# et Automatique)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shared utils for network communication components."""
+
+import secrets
+
+__all__ = [
+ "create_secret_token",
+]
+
+
+DEFAULT_TOKEN_NBYTES = 64 # Default number of bytes in user auth tokens.
+
+
+def create_secret_token() -> str:
+ """Create a random secret string token."""
+ return secrets.token_hex(nbytes=DEFAULT_TOKEN_NBYTES)
diff --git a/test/communication3/server/test_files.py b/test/communication3/server/test_files.py
new file mode 100644
index 0000000000000000000000000000000000000000..024a4c2be272ee1f5449eca60506842be00231dc
--- /dev/null
+++ b/test/communication3/server/test_files.py
@@ -0,0 +1,262 @@
+# coding: utf-8
+
+# Copyright 2023 Inria (Institut National de Recherche en Informatique
+# et Automatique)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for the FilesManager communication server component."""
+
+import datetime
+import os
+import tempfile
+import time
+from typing import Iterator, Optional
+from unittest import mock
+
+import pytest
+
+from declearn.communication3.server import (
+ FileAuthError,
+ FileExpiredError,
+ FilesManager,
+)
+
+
+@pytest.fixture(name="folder")
+def folder_fixture() -> Iterator[str]:
+ """Provide with a temporary folder."""
+ with tempfile.TemporaryDirectory() as folder:
+ yield folder
+
+
+class TestFilesManager:
+ """Unit tests for the FilesManager communication server component."""
+
+ @pytest.mark.parametrize(
+ "prefix", [None, "prefix"], ids=["noprefix", "prefixed"]
+ )
+ def test_create_file(
+ self,
+ folder: str,
+ prefix: Optional[str],
+ ) -> None:
+ """Test that `create_file` properly creates a file."""
+ # Setup a FilesManager and some mock file content.
+ manager = FilesManager(folder)
+ content = mock.create_autospec(bytes, instance=True)
+ # Call the method, mocking file opening.
+ with mock.patch("builtins.open", mock.mock_open()) as patch_open:
+ name, access_token, delete_token = manager.create_file(
+ content, prefix=prefix
+ )
+ # Verify that a file was opened and that outputs match expectations.
+ patch_open.assert_called_once()
+ patch_open.return_value.write.assert_called_once_with(content)
+ assert isinstance(name, str)
+ if prefix:
+ assert name.startswith(prefix)
+ assert isinstance(access_token, str)
+ assert isinstance(delete_token, str)
+ assert access_token != delete_token
+
+ def test_access_file(
+ self,
+ folder: str,
+ ) -> None:
+ """Test that `access_files` properly streams the file."""
+ # Setup a FilesManager and create a file.
+ manager = FilesManager(folder)
+ content = b"placeholder"
+ name, access_token, _ = manager.create_file(content)
+ # Verify that the file can properly be accessed.
+ outputs = b"".join(manager.access_file(name, access_token))
+ assert outputs == content
+
+ def test_access_file_with_wrong_token(
+ self,
+ folder: str,
+ ) -> None:
+ """Test that `access_files` raises on wrong input token."""
+ # Setup a FilesManager and create a file.
+ manager = FilesManager(folder)
+ content = b"placeholder"
+ name, _, delete_token = manager.create_file(content)
+ # Verify that the expected error is raised with a wrongful token.
+ with pytest.raises(FileAuthError):
+ manager.access_file(name, delete_token)
+
+ def test_access_file_with_wrong_name(
+ self,
+ folder: str,
+ ) -> None:
+ """Test that `access_files` raises on wrong input name."""
+ # Setup a FilesManager with no file.
+ manager = FilesManager(folder)
+ # Verify that the expected error is raised with a wrongful name.
+ with pytest.raises(FileNotFoundError):
+ manager.access_file("fake-name", "fake-token")
+
+ def test_access_expired_file_too_many_accesses(
+ self,
+ folder: str,
+ ) -> None:
+ """Test that files can expire based on specified number of accesses."""
+ # Setup a FilesManager with a single-access file.
+ manager = FilesManager(folder)
+ content = b"placeholder"
+ name, access_token, _ = manager.create_file(content, max_accesses=1)
+ # Test that the file can be accessed once and only once.
+ assert b"".join(manager.access_file(name, access_token)) == content
+ with pytest.raises(FileExpiredError):
+ manager.access_file(name, access_token)
+
+ def test_access_expired_file_too_long_timekeep(
+ self,
+ folder: str,
+ ) -> None:
+ """Test that files can expire based on specified storage duration."""
+ # Setup a FilesManager with a 0.2-second storage duration.
+ manager = FilesManager(folder)
+ content = b"placeholder"
+ name, access_token, _ = manager.create_file(
+ content, max_keeptime=datetime.timedelta(milliseconds=200)
+ )
+ # Test that the file can be accessed prior to the limit.
+ assert b"".join(manager.access_file(name, access_token)) == content
+ # Test that the file cannot be accessed past the limit.
+ time.sleep(0.2)
+ with pytest.raises(FileExpiredError):
+ manager.access_file(name, access_token)
+
+ def test_delete_file(
+ self,
+ folder: str,
+ ) -> None:
+ """Test that `delete_file` properly deletes a stored file."""
+ # Setup a FilesManager with a single file.
+ manager = FilesManager(folder)
+ content = b"placeholder"
+ name, _, delete_token = manager.create_file(content)
+ # Test that the file can be deleted (and is effectively removed).
+ assert len(os.listdir(folder)) == 1
+ manager.delete_file(name, delete_token)
+ assert len(os.listdir(folder)) == 0
+
+ def test_delete_file_with_wrong_token(
+ self,
+ folder: str,
+ ) -> None:
+ """Test that `delete_files` raises on wrong input token."""
+ # Setup a FilesManager and create a file.
+ manager = FilesManager(folder)
+ content = b"placeholder"
+ name, access_token, _ = manager.create_file(content)
+ # Verify that the expected error is raised with a wrongful token.
+ with pytest.raises(FileAuthError):
+ manager.delete_file(name, access_token)
+
+ def test_delete_file_with_wrong_name(
+ self,
+ folder: str,
+ ) -> None:
+ """Test that `delete_files` raises on wrong input name."""
+ # Setup a FilesManager with no file.
+ manager = FilesManager(folder)
+ # Verify that the expected error is raised with a wrongful name.
+ with pytest.raises(FileNotFoundError):
+ manager.delete_file("fake-name", "fake-token")
+
+ def test_delete_file_redundant_calls_no_deref(
+ self,
+ folder: str,
+ ) -> None:
+ """Test that redundant calls to `delete_file` pass quietly."""
+ # Setup a FilesManager with a single file.
+ manager = FilesManager(folder)
+ content = b"placeholder"
+ name, _, delete_token = manager.create_file(content)
+ # Test that the file can be deleted twice - i.e. that deletion
+ # is accepted even for deprecated/no-longer-existing files.
+ manager.delete_file(name, delete_token)
+ manager.delete_file(name, delete_token)
+
+ def test_delete_file_redundant_calls_with_deref(
+ self,
+ folder: str,
+ ) -> None:
+ """Test that redundant calls to `delete_file` raise if `deref=True`."""
+ # Setup a FilesManager with a single file.
+ manager = FilesManager(folder)
+ content = b"placeholder"
+ name, _, delete_token = manager.create_file(content)
+ # Test that the file can only be deleted once - due to its
+ # dereferencing, that make it unknown to the manager.
+ manager.delete_file(name, delete_token, deref=True)
+ with pytest.raises(FileNotFoundError):
+ manager.delete_file(name, delete_token)
+
+ def test_delete_file_during_access(
+ self,
+ folder: str,
+ ) -> None:
+ """Test that `delete_file` and `access_file` play along nicely."""
+ # Setup a FilesManager with a single file.
+ manager = FilesManager(folder)
+ content = b"placeholder"
+ name, access_token, delete_token = manager.create_file(content)
+ # Open an access to the file, then trigger its deletion.
+ fstream = manager.access_file(name, access_token)
+ manager.delete_file(name, delete_token)
+ # Verify that new accesses are refused, due to the planned deletion.
+ with pytest.raises(FileExpiredError):
+ manager.access_file(name, access_token)
+ # Verify that the file exists and the pre-existing stream works.
+ assert len(os.listdir(folder)) == 1
+ assert b"".join(fstream) == content
+ # Verify that the file is deleted once the open reader is done.
+ assert len(os.listdir(folder)) == 0
+
+ def test_remove_expired_references(
+ self,
+ folder: str,
+ ) -> None:
+ """Test that `remove_expired_references` properly collects files."""
+ # Setup a FilesManager with four files:
+ # - a is bound to remain in-place
+ # - b is deleted after creation
+ # - c may expire but is left unaccessed
+ # - d expires right away due to zero-seconds storage duration
+ manager = FilesManager(folder)
+ content = b"placeholder"
+ file_a = manager.create_file(content)
+ file_b = manager.create_file(content)
+ file_c = manager.create_file(content, max_accesses=1)
+ file_d = manager.create_file(
+ content, max_keeptime=datetime.timedelta(0)
+ )
+ assert len(os.listdir(folder)) == 4
+ manager.delete_file(name=file_b[0], token=file_b[2])
+ assert len(os.listdir(folder)) == 3
+ # Call the method.
+ manager.remove_expired_references()
+ # Verify that only two files remain (a and c).
+ assert len(os.listdir(folder)) == 2
+ # Verify that a and c are still referenced.
+ manager.access_file(name=file_a[0], token=file_a[1])
+ manager.access_file(name=file_c[0], token=file_c[1])
+ # Verify that b and d are no longer referenced.
+ with pytest.raises(FileNotFoundError):
+ manager.access_file(name=file_b[0], token=file_b[1])
+ with pytest.raises(FileNotFoundError):
+ manager.access_file(name=file_d[0], token=file_d[1])