operations/files: add atime and mtime arguments to files.put operation

Co-authored-by: Nick Mills-Barrett <[email protected]>

Author: vram0gh2

pyinfra/facts/files.py

@@ -28,7 +28,7 @@
 
 STAT_REGEX = (
     r"user=(.*) group=(.*) mode=(.*) "
-    r"atime=([0-9]*) mtime=([0-9]*) ctime=([0-9]*) "
+    r"atime=(-?[0-9]*) mtime=(-?[0-9]*) ctime=(-?[0-9]*) "
     r"size=([0-9]*) (.*)"
 )
 

pyinfra/operations/files.py

@@ -8,7 +8,7 @@
 import posixpath
 import sys
 import traceback
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
 from fnmatch import fnmatch
 from io import StringIO
 from pathlib import Path
@@ -59,6 +59,7 @@
 
 from .util import files as file_utils
 from .util.files import (
+    MetadataTimeField,
     adjust_regex,
     ensure_mode_int,
     get_timestamp,
@@ -813,6 +814,56 @@ def get(
             host.noop("file {0} has already been downloaded".format(dest))
 
 
+def _canonicalize_timespec(field: MetadataTimeField, local_file, timespec):
+    if isinstance(timespec, datetime):
+        if not timespec.tzinfo:
+            # specify remote host timezone
+            timespec_with_tz = timespec.replace(tzinfo=host.get_fact(Date).tzinfo)
+            return timespec_with_tz
+        else:
+            return timespec
+    elif isinstance(timespec, bool) and timespec:
+        lf_ts = (
+            os.stat(local_file).st_atime
+            if field is MetadataTimeField.ATIME
+            else os.stat(local_file).st_mtime
+        )
+        return datetime.fromtimestamp(lf_ts, tz=timezone.utc)
+    else:
+        try:
+            isodatetime = datetime.fromisoformat(timespec)
+            if not isodatetime.tzinfo:
+                return isodatetime.replace(tzinfo=host.get_fact(Date).tzinfo)
+            else:
+                return isodatetime
+        except ValueError:
+            try:
+                timestamp = float(timespec)
+                return datetime.fromtimestamp(timestamp, tz=timezone.utc)
+            except ValueError:
+                # verify there is a remote file matching path in timesrc
+                ref_file = host.get_fact(File, path=timespec)
+                if ref_file:
+                    if field is MetadataTimeField.ATIME:
+                        assert ref_file["atime"] is not None
+                        return ref_file["atime"].replace(tzinfo=timezone.utc)
+                    else:
+                        assert ref_file["mtime"] is not None
+                        return ref_file["mtime"].replace(tzinfo=timezone.utc)
+                else:
+                    ValueError("Bad argument for `timesspec`: {0}".format(timespec))
+
+
+# returns True for a visible difference in the second field between the datetime values
+#   in the ref's TZ
+def _times_differ_in_s(ref, cand):
+    assert ref.tzinfo and cand.tzinfo
+    cand_in_ref_tz = cand.astimezone(ref.tzinfo)
+    return (abs((cand_in_ref_tz - ref).total_seconds()) >= 1.0) or (
+        ref.second != cand_in_ref_tz.second
+    )
+
+
 @operation()
 def put(
     src: str | IO[Any],
@@ -824,6 +875,8 @@ def put(
     create_remote_dir=True,
     force=False,
     assume_exists=False,
+    atime: datetime | float | int | str | bool | None = None,
+    mtime: datetime | float | int | str | bool | None = None,
 ):
     """
     Upload a local file, or file-like object, to the remote system.
@@ -837,6 +890,8 @@ def put(
     + create_remote_dir: create the remote directory if it doesn't exist
     + force: always upload the file, even if the remote copy matches
     + assume_exists: whether to assume the local file exists
+    + atime: value of atime the file should have, use ``True`` to match the local file
+    + mtime: value of mtime the file should have, use ``True`` to match the local file
 
     ``dest``:
         If this is a directory that already exists on the remote side, the local
@@ -853,7 +908,21 @@ def put(
         user & group as passed to ``files.put``. The mode will *not* be copied over,
         if this is required call ``files.directory`` separately.
 
-    Note:
+    ``atime`` and ``mtime``:
+        When set to values other than ``False`` or ``None``, the respective metadata
+        fields on the remote file will updated accordingly.  Timestamp values are
+        considered equivalent if the difference is less than one second and they have
+        the identical number in the seconds field.  If set to ``True`` the local
+        file is the source of the value.  Otherwise, these values can be provided as
+        ``datetime`` objects, POSIX timestamps, or strings that can be parsed into
+        either of these date and time specifications.  They can also be reference file
+        paths on the remote host, as with the ``-r`` argument to ``touch``.  If a
+        ``datetime`` argument has no ``tzinfo`` value (i.e., it is naive), it is
+        assumed to be in the remote host's local timezone.  There is no shortcut for
+        setting both ``atime` and ``mtime`` values with a single time specification,
+        unlike the native ``touch`` command.
+
+    Notes:
         This operation is not suitable for large files as it may involve copying
         the file before uploading it.
 
@@ -862,6 +931,12 @@ def put(
         behave as if the remote file does not match the specified permissions and
         requires a change.
 
+        If the ``atime`` argument is set for a given file, unless the remote
+        filesystem is mounted ``noatime`` or ``relatime``, multiple runs of this
+        operation will trigger the change detection for that file, since the act of
+        reading and checksumming the file will cause the host OS to update the file's
+        ``atime``.
+
     **Examples:**
 
     .. code:: python
@@ -937,7 +1012,22 @@ def put(
         if mode:
             yield file_utils.chmod(dest, mode)
 
-    # File exists, check sum and check user/group/mode if supplied
+        # do mtime before atime to ensure atime setting isn't undone by mtime setting
+        if mtime:
+            yield file_utils.touch(
+                dest,
+                MetadataTimeField.MTIME,
+                _canonicalize_timespec(MetadataTimeField.MTIME, src, mtime),
+            )
+
+        if atime:
+            yield file_utils.touch(
+                dest,
+                MetadataTimeField.ATIME,
+                _canonicalize_timespec(MetadataTimeField.ATIME, src, atime),
+            )
+
+    # File exists, check sum and check user/group/mode/atime/mtime if supplied
     else:
         if not _file_equal(local_sum_path, dest):
             yield FileUploadCommand(
@@ -952,6 +1042,20 @@ def put(
             if mode:
                 yield file_utils.chmod(dest, mode)
 
+            if mtime:
+                yield file_utils.touch(
+                    dest,
+                    MetadataTimeField.MTIME,
+                    _canonicalize_timespec(MetadataTimeField.MTIME, src, mtime),
+                )
+
+            if atime:
+                yield file_utils.touch(
+                    dest,
+                    MetadataTimeField.ATIME,
+                    _canonicalize_timespec(MetadataTimeField.ATIME, src, atime),
+                )
+
         else:
             changed = False
 
@@ -965,6 +1069,26 @@ def put(
                 yield file_utils.chown(dest, user, group)
                 changed = True
 
+            # Check mtime
+            if mtime:
+                canonical_mtime = _canonicalize_timespec(MetadataTimeField.MTIME, src, mtime)
+                assert remote_file["mtime"] is not None
+                if _times_differ_in_s(
+                    canonical_mtime, remote_file["mtime"].replace(tzinfo=timezone.utc)
+                ):
+                    yield file_utils.touch(dest, MetadataTimeField.MTIME, canonical_mtime)
+                    changed = True
+
+            # Check atime
+            if atime:
+                canonical_atime = _canonicalize_timespec(MetadataTimeField.ATIME, src, atime)
+                assert remote_file["atime"] is not None
+                if _times_differ_in_s(
+                    canonical_atime, remote_file["atime"].replace(tzinfo=timezone.utc)
+                ):
+                    yield file_utils.touch(dest, MetadataTimeField.ATIME, canonical_atime)
+                    changed = True
+
             if not changed:
                 host.noop("file {0} is already uploaded".format(dest))
 

pyinfra/operations/util/files.py

@@ -1,12 +1,18 @@
 from __future__ import annotations
 
 import re
-from datetime import datetime
+from datetime import datetime, timezone
+from enum import Enum
 from typing import Callable
 
 from pyinfra.api import QuoteString, StringCommand
 
 
+class MetadataTimeField(Enum):
+    ATIME = "atime"
+    MTIME = "mtime"
+
+
 def unix_path_join(*parts) -> str:
     part_list = list(parts)
     part_list[0:-1] = [part.rstrip("/") for part in part_list[0:-1]]
@@ -156,6 +162,34 @@ def chown(
     return StringCommand(" ".join(args), user_group, QuoteString(target))
 
 
+# like the touch command, but only supports setting one field at a time, and expects any
+#   reference times to have been read from the reference file metadata and turned into
+#   aware datetimes
+def touch(
+    target: str,
+    timefield: MetadataTimeField,
+    timesrc: datetime,
+    dereference=True,
+) -> StringCommand:
+    args = ["touch"]
+
+    if timefield is MetadataTimeField.ATIME:
+        args.append("-a")
+    else:
+        args.append("-m")
+
+    if not dereference:
+        args.append("-h")
+
+    # don't reinvent the wheel; use isoformat()
+    timestr = timesrc.astimezone(timezone.utc).isoformat()
+    # but replace the ISO format TZ offset with "Z" for BSD
+    timestr = timestr.replace("+00:00", "Z")
+    args.extend(["-d", timestr])
+
+    return StringCommand(" ".join(args), QuoteString(target))
+
+
 def adjust_regex(line: str, escape_regex_characters: bool) -> str:
     """
     Ensure the regex starts with '^' and ends with '$' and escape regex characters if requested

tests/operations/files.put/atime_mtime_sub_second.json

@@ -0,0 +1,31 @@
+{
+    "args": ["somefile.txt", "/home/somefile.txt"],
+    "kwargs": {
+        "atime": "datetime:2002-09-15T10:11:12+00:00",
+        "mtime": "datetime:2002-09-15T10:11:11.888888+00:00"
+    },
+    "local_files": {
+        "files": {
+            "somefile.txt": null
+        },
+        "dirs": {}
+    },
+    "facts": {
+        "files.File": {
+            "path=/home/somefile.txt": {
+                "mode": 640,
+                "atime": "datetime:2002-09-15T10:11:12",
+                "mtime": "datetime:2002-09-15T10:11:12"
+            }
+        },
+        "files.Directory": {
+            "path=/home": true
+        },
+        "files.Sha1File": {
+            "path=/home/somefile.txt": "ac2cd59a622114712b5b21081763c54bf0caacb8"
+        }
+    },
+    "commands": [
+        "touch -m -d 2002-09-15T10:11:11.888888Z /home/somefile.txt"
+    ]
+}

tests/operations/files.put/change_atime_eq_mtime_ref_remote.json

@@ -0,0 +1,37 @@
+{
+    "args": ["somefile.txt", "/home/somefile.txt"],
+    "kwargs": {
+        "atime": "/var/timeref.blob",
+        "mtime": "/var/timeref.blob",
+    },
+    "local_files": {
+        "files": {
+            "somefile.txt": null
+        },
+        "dirs": {}
+    },
+    "facts": {
+        "files.File": {
+            "path=/home/somefile.txt": {
+                "mode": 500,
+                "atime": "datetime:2020-07-15T09:19:27",
+                "mtime": "datetime:2002-09-15T10:11:12"
+            },
+            "path=/var/timeref.blob": {
+                "mode": 644,
+                "atime": "datetime:1991-04-15T18:18:27",
+                "mtime": "datetime:2002-09-15T10:11:12"
+            },
+        },
+        "files.Directory": {
+            "path=/home": true,
+            "path=/var": true
+        },
+        "files.Sha1File": {
+            "path=/home/somefile.txt": "ac2cd59a622114712b5b21081763c54bf0caacb8"
+        }
+    },
+    "commands": [
+        "touch -a -d 1991-04-15T18:18:27Z /home/somefile.txt",
+    ]
+}

tests/operations/files.put/change_mtime_arg_no_tz.json

@@ -0,0 +1,37 @@
+{
+    "args": ["somefile.txt", "/home/somefile.txt"],
+    "kwargs": {
+        "atime": false,
+        "mtime": "2022-11-17T20:45:00",
+    },
+    "local_files": {
+        "files": {
+            "somefile.txt": {
+                "mode": 644,
+                "ctime": "datetime:1997-04-21T18:06:55.982Z",
+                "atime": "datetime:2020-06-20T22:09:17",
+                "mtime": "datetime:2000-05-01T00:01:00Z"
+            },
+        },
+        "dirs": {}
+    },
+    "facts": {
+        "files.File": {
+            "path=/home/somefile.txt": {
+                "mode": 500,
+                "atime": "datetime:2020-07-15T09:19:27",
+                "mtime": "datetime:2020-07-15T09:19:27"
+            }
+        },
+        "files.Directory": {
+            "path=/home": true
+        },
+        "files.Sha1File": {
+            "path=/home/somefile.txt": "ac2cd59a622114712b5b21081763c54bf0caacb8"
+        },
+        "server.Date": "datetime:2017-03-03T10:07:37-06:00"
+    },
+    "commands": [
+        "touch -m -d 2022-11-18T02:45:00Z /home/somefile.txt",
+    ]
+}