feat: support preserving symlinks when copying templates (#938)

* feat: Preserve symlinks when copying templates

* test: Add tests for symlink copying

copier/main.py

@@ -11,7 +11,7 @@ from functools import partial
 from itertools import chain
 from pathlib import Path
 from shutil import rmtree
-from typing import Callable, Iterable, Mapping, Optional, Sequence
+from typing import Callable, Iterable, Mapping, Optional, Sequence, Union
 from unicodedata import normalize
 
 from jinja2.loaders import FileSystemLoader
@@ -29,7 +29,7 @@ from questionary import unsafe_prompt
 from .errors import CopierAnswersInterrupt, ExtensionNotFoundError, UserMessageError
 from .subproject import Subproject
 from .template import Task, Template
-from .tools import Style, TemporaryDirectory, printf
+from .tools import Style, TemporaryDirectory, printf, readlink
 from .types import (
     AnyByStrDict,
     JSONSerializable,
@@ -302,7 +302,11 @@ class Worker:
         return bool(ask(f" Overwrite {dst_relpath}?", default=True))
 
     def _render_allowed(
-        self, dst_relpath: Path, is_dir: bool = False, expected_contents: bytes = b""
+        self,
+        dst_relpath: Path,
+        is_dir: bool = False,
+        is_symlink: bool = False,
+        expected_contents: Union[bytes, Path] = b"",
     ) -> bool:
         """Determine if a file or directory can be rendered.
 
@@ -311,6 +315,8 @@ class Worker:
                 Relative path to destination.
             is_dir:
                 Indicate if the path must be treated as a directory or not.
+            is_symlink:
+                Indicate if the path must be treated as a symlink or not.
             expected_contents:
                 Used to compare existing file contents with them. Allows to know if
                 rendering is needed.
@@ -321,7 +327,11 @@ class Worker:
         if dst_relpath != Path(".") and self.match_exclude(dst_relpath):
             return False
         try:
-            previous_content = dst_abspath.read_bytes()
+            previous_content: Union[bytes, Path]
+            if is_symlink:
+                previous_content = readlink(dst_abspath)
+            else:
+                previous_content = dst_abspath.read_bytes()
         except FileNotFoundError:
             printf(
                 "create",
@@ -519,6 +529,45 @@ class Worker:
             dst_abspath.write_bytes(new_content)
             dst_abspath.chmod(src_mode)
 
+    def _render_symlink(self, src_abspath: Path) -> None:
+        """Render one symlink.
+
+        Args:
+            src_abspath:
+                Symlink to be rendered. It must be an absolute path within
+                the template.
+        """
+        assert src_abspath.is_absolute()
+        src_relpath = src_abspath.relative_to(self.template_copy_root)
+        dst_relpath = self._render_path(src_relpath)
+        if dst_relpath is None:
+            return
+        dst_abspath = Path(self.subproject.local_abspath, dst_relpath)
+
+        src_target = readlink(src_abspath)
+        if src_abspath.name.endswith(self.template.templates_suffix):
+            dst_target = Path(self._render_string(str(src_target)))
+        else:
+            dst_target = src_target
+
+        if not self._render_allowed(
+            dst_relpath,
+            expected_contents=dst_target,
+            is_symlink=True,
+        ):
+            return
+
+        if not self.pretend:
+            # symlink_to doesn't overwrite existing files, so delete it first
+            if dst_abspath.is_symlink() or dst_abspath.exists():
+                dst_abspath.unlink()
+            dst_abspath.symlink_to(dst_target)
+            if sys.platform == "darwin":
+                # Only macOS supports permissions on symlinks.
+                # Other platforms just copy the permission of the target
+                src_mode = src_abspath.lstat().st_mode
+                dst_abspath.lchmod(src_mode)
+
     def _render_folder(self, src_abspath: Path) -> None:
         """Recursively render a folder.
 
@@ -540,6 +589,8 @@ class Worker:
         for file in src_abspath.iterdir():
             if file.is_dir():
                 self._render_folder(file)
+            elif file.is_symlink() and self.template.preserve_symlinks:
+                self._render_symlink(file)
             else:
                 self._render_file(file)
 

copier/template.py

@@ -446,6 +446,14 @@ class Template:
             return DEFAULT_TEMPLATES_SUFFIX
         return result
 
+    @cached_property
+    def preserve_symlinks(self) -> bool:
+        """Know if Copier should preserve symlinks when rendering the template.
+
+        See [preserve_symlinks][].
+        """
+        return bool(self.config_data.get("preserve_symlinks", False))
+
     @cached_property
     def local_abspath(self) -> Path:
         """Get the absolute path to the template on disk.

copier/tools.py

@@ -178,3 +178,17 @@ class TemporaryDirectory(tempfile.TemporaryDirectory):
     @staticmethod
     def _robust_cleanup(name):
         shutil.rmtree(name, ignore_errors=False, onerror=handle_remove_readonly)
+
+
+def readlink(link: Path) -> Path:
+    """A custom version of os.readlink/pathlib.Path.readlink.
+
+    pathlib.Path.readlink is what we ideally would want to use, but it is only available on python>=3.9.
+    os.readlink doesn't support Path and bytes on Windows for python<3.8
+    """
+    if sys.version_info >= (3, 9):
+        return link.readlink()
+    elif sys.version_info >= (3, 8) or os.name != "nt":
+        return Path(os.readlink(link))
+    else:
+        return Path(os.readlink(str(link)))

docs/configuring.md

@@ -1022,6 +1022,18 @@ Run but do not make any changes.
 
     Not supported in `copier.yml`.
 
+### `preserve_symlinks`
+
+-   Format: `bool`
+-   CLI flags: N/A
+-   Default value: `False`
+
+Keep symlinks as symlinks. If this is set to `False` symlinks will be replaced with the
+file they point to.
+
+When set to `True` and the symlink ends with the template suffix (`.jinja` by default)
+the target path of the symlink will be rendered as a jinja template.
+
 ### `quiet`
 
 -   Format: `bool`

tests/demo/symlink.txt

@@ -0,0 +1 @@
+aaaa.txt

tests/helpers.py

@@ -6,7 +6,7 @@ import textwrap
 from enum import Enum
 from hashlib import sha1
 from pathlib import Path
-from typing import Mapping, Optional, Tuple, Union, cast
+from typing import Mapping, Optional, Tuple, Union
 
 from pexpect.popen_spawn import PopenSpawn
 from plumbum import local
@@ -100,18 +100,30 @@ def assert_file(tmp_path: Path, *path: str) -> None:
 
 
 def build_file_tree(
-    spec: Mapping[StrOrPath, Union[str, bytes]], dedent: bool = True
+    spec: Mapping[StrOrPath, Union[str, bytes, Path]], dedent: bool = True
-) -> None:
+):
-    """Builds a file tree based on the received spec."""
+    """Builds a file tree based on the received spec.
+
+    Params:
+        spec:
+            A mapping from filesystem paths to file contents. If the content is
+            a Path object a symlink to the path will be created instead.
+
+        dedent: Dedent file contents.
+    """
     for path, contents in spec.items():
         path = Path(path)
-        binary = isinstance(contents, bytes)
-        if not binary and dedent:
-            contents = textwrap.dedent(cast(str, contents))
         path.parent.mkdir(parents=True, exist_ok=True)
-        mode = "wb" if binary else "w"
+        if isinstance(contents, Path):
-        with path.open(mode) as fd:
+            os.symlink(str(contents), path)
-            fd.write(contents)
+        else:
+            binary = isinstance(contents, bytes)
+            if not binary and dedent:
+                assert isinstance(contents, str)
+                contents = textwrap.dedent(contents)
+            mode = "wb" if binary else "w"
+            with path.open(mode) as fd:
+                fd.write(contents)
 
 
 def expect_prompt(

tests/test_copy.py

@@ -135,6 +135,8 @@ def test_copy(tmp_path: Path) -> None:
     assert not (tmp_path / "py2_folder" / "thing.py").exists()
     assert (tmp_path / "py3_folder" / "thing.py").exists()
 
+    assert (tmp_path / "aaaa.txt").exists()
+
 
 @pytest.mark.impure
 def test_copy_repo(tmp_path: Path) -> None:
@@ -288,22 +290,28 @@ def test_empty_dir(tmp_path_factory: pytest.TempPathFactory, generate: bool) ->
             ),
             (src / "tpl" / "two.txt"): "[[ do_it ]]",
             (src / "tpl" / "[% if do_it %]three.txt[% endif %].jinja"): "[[ do_it ]]",
-            (src / "tpl" / "four" / "[% if do_it %]five.txt[% endif %].jinja"): (
+            (src / "tpl" / "[% if do_it %]four.txt[% endif %].jinja"): Path(
+                "[% if do_it %]three.txt[% endif %].jinja"
+            ),
+            (src / "tpl" / "five" / "[% if do_it %]six.txt[% endif %].jinja"): (
                 "[[ do_it ]]"
             ),
         },
     )
     copier.run_copy(str(src), dst, {"do_it": generate}, defaults=True, overwrite=True)
-    assert (dst / "four").is_dir()
+    assert (dst / "five").is_dir()
     assert (dst / "two.txt").read_text() == "[[ do_it ]]"
     assert (dst / "one_dir").exists() == generate
     assert (dst / "three.txt").exists() == generate
+    assert (dst / "four.txt").exists() == generate
     assert (dst / "one_dir").is_dir() == generate
     assert (dst / "one_dir" / "one.txt").is_file() == generate
     if generate:
         assert (dst / "one_dir" / "one.txt").read_text() == repr(generate)
         assert (dst / "three.txt").read_text() == repr(generate)
-        assert (dst / "four" / "five.txt").read_text() == repr(generate)
+        assert not (dst / "four.txt").is_symlink()
+        assert (dst / "four.txt").read_text() == repr(generate)
+        assert (dst / "five" / "six.txt").read_text() == repr(generate)
 
 
 @pytest.mark.skipif(

tests/test_dirty_local.py

@@ -66,6 +66,7 @@ def test_update(tmp_path_factory: pytest.TempPathFactory) -> None:
                 delete me.
                 """
             ),
+            (src / "symlink.txt"): Path("./to_delete.txt"),
         }
     )
 
@@ -84,6 +85,10 @@ def test_update(tmp_path_factory: pytest.TempPathFactory) -> None:
         with open("aaaa.txt", "a") as f:
             f.write("dolor sit amet")
 
+        # test updating a symlink
+        Path("symlink.txt").unlink()
+        Path("symlink.txt").symlink_to("test_file.txt")
+
         # test removing a file
         Path("to_delete.txt").unlink()
 
@@ -98,6 +103,10 @@ def test_update(tmp_path_factory: pytest.TempPathFactory) -> None:
 
     assert (src / "aaaa.txt").read_text() != (dst / "aaaa.txt").read_text()
 
+    p1 = src / "symlink.txt"
+    p2 = dst / "symlink.txt"
+    assert p1.read_text() != p2.read_text()
+
     assert (dst / "to_delete.txt").exists()
 
     with pytest.warns(DirtyLocalWarning):
@@ -108,6 +117,11 @@ def test_update(tmp_path_factory: pytest.TempPathFactory) -> None:
 
     assert (src / "aaaa.txt").read_text() == (dst / "aaaa.txt").read_text()
 
+    p1 = src / "symlink.txt"
+    p2 = dst / "symlink.txt"
+    assert p1.read_text() == p2.read_text()
+    assert not (dst / "symlink.txt").is_symlink()
+
     # HACK https://github.com/copier-org/copier/issues/461
     # TODO test file deletion on update
     # assert not (dst / "to_delete.txt").exists()

tests/test_symlinks.py

@@ -0,0 +1,387 @@
+import os
+from pathlib import Path
+
+import pytest
+from plumbum import local
+from plumbum.cmd import git
+
+from copier import copy, readlink, run_copy, run_update
+from copier.errors import DirtyLocalWarning
+
+from .helpers import build_file_tree
+
+
+def test_copy_symlink(tmp_path_factory):
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    # Prepare repo bundle
+    repo = src / "repo"
+    repo.mkdir()
+    build_file_tree(
+        {
+            repo
+            / "copier.yaml": """\
+            _preserve_symlinks: true
+            """,
+            repo / "target.txt": "Symlink target",
+            repo / "symlink.txt": Path("target.txt"),
+        }
+    )
+
+    with local.cwd(src):
+        git("init")
+        git("add", ".")
+        git("commit", "-m", "hello world")
+
+    copy(
+        str(repo),
+        dst,
+        defaults=True,
+        overwrite=True,
+        vcs_ref="HEAD",
+    )
+
+    assert os.path.exists(dst / "target.txt")
+    assert os.path.exists(dst / "symlink.txt")
+    assert os.path.islink(dst / "symlink.txt")
+    assert readlink(dst / "symlink.txt") == Path("target.txt")
+
+
+def test_copy_symlink_templated_name(tmp_path_factory):
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    # Prepare repo bundle
+    repo = src / "repo"
+    repo.mkdir()
+    build_file_tree(
+        {
+            repo
+            / "copier.yaml": """\
+            _preserve_symlinks: true
+            symlink_name: symlink
+            """,
+            repo / "target.txt": "Symlink target",
+            repo / "{{ symlink_name }}.txt": Path("target.txt"),
+        }
+    )
+
+    with local.cwd(src):
+        git("init")
+        git("add", ".")
+        git("commit", "-m", "hello world")
+
+    copy(
+        str(repo),
+        dst,
+        defaults=True,
+        overwrite=True,
+        vcs_ref="HEAD",
+    )
+
+    assert os.path.exists(dst / "target.txt")
+    assert os.path.exists(dst / "symlink.txt")
+    assert os.path.islink(dst / "symlink.txt")
+    assert readlink(dst / "symlink.txt") == Path("target.txt")
+
+
+def test_copy_symlink_templated_target(tmp_path_factory):
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    # Prepare repo bundle
+    repo = src / "repo"
+    repo.mkdir()
+    build_file_tree(
+        {
+            repo
+            / "copier.yaml": """\
+            _preserve_symlinks: true
+            target_name: target
+            """,
+            repo / "{{ target_name }}.txt": "Symlink target",
+            repo / "symlink1.txt.jinja": Path("{{ target_name }}.txt"),
+            repo / "symlink2.txt": Path("{{ target_name }}.txt"),
+        }
+    )
+
+    with local.cwd(src):
+        git("init")
+        git("add", ".")
+        git("commit", "-m", "hello world")
+
+    copy(
+        str(repo),
+        dst,
+        defaults=True,
+        overwrite=True,
+        vcs_ref="HEAD",
+    )
+
+    assert os.path.exists(dst / "target.txt")
+
+    assert os.path.exists(dst / "symlink1.txt")
+    assert os.path.islink(dst / "symlink1.txt")
+    assert readlink(dst / "symlink1.txt") == Path("target.txt")
+
+    assert not os.path.exists(dst / "symlink2.txt")
+    assert os.path.islink(dst / "symlink2.txt")
+    assert readlink(dst / "symlink2.txt") == Path("{{ target_name }}.txt")
+
+
+def test_copy_symlink_missing_target(tmp_path_factory):
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    # Prepare repo bundle
+    repo = src / "repo"
+    repo.mkdir()
+    build_file_tree(
+        {
+            repo
+            / "copier.yaml": """\
+            _preserve_symlinks: true
+            """,
+            repo / "symlink.txt": Path("target.txt"),
+        }
+    )
+
+    with local.cwd(src):
+        git("init")
+        git("add", ".")
+        git("commit", "-m", "hello world")
+
+    copy(
+        str(repo),
+        dst,
+        defaults=True,
+        overwrite=True,
+        vcs_ref="HEAD",
+    )
+
+    assert os.path.islink(dst / "symlink.txt")
+    assert readlink(dst / "symlink.txt") == Path("target.txt")
+    assert not os.path.exists(
+        dst / "symlink.txt"
+    )  # exists follows symlinks, It returns False as the target doesn't exist
+
+
+def test_option_preserve_symlinks_false(tmp_path_factory):
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    # Prepare repo bundle
+    repo = src / "repo"
+    repo.mkdir()
+    build_file_tree(
+        {
+            repo
+            / "copier.yaml": """\
+            _preserve_symlinks: false
+            """,
+            repo / "target.txt": "Symlink target",
+            repo / "symlink.txt": Path("target.txt"),
+        }
+    )
+
+    with local.cwd(src):
+        git("init")
+        git("add", ".")
+        git("commit", "-m", "hello world")
+
+    copy(
+        str(repo),
+        dst,
+        defaults=True,
+        overwrite=True,
+        vcs_ref="HEAD",
+    )
+
+    assert os.path.exists(dst / "target.txt")
+    assert os.path.exists(dst / "symlink.txt")
+    assert not os.path.islink(dst / "symlink.txt")
+
+
+def test_option_preserve_symlinks_default(tmp_path_factory):
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    # Prepare repo bundle
+    repo = src / "repo"
+    repo.mkdir()
+    build_file_tree(
+        {
+            repo
+            / "copier.yaml": """\
+            """,
+            repo / "target.txt": "Symlink target",
+            repo / "symlink.txt": Path("target.txt"),
+        }
+    )
+
+    with local.cwd(src):
+        git("init")
+        git("add", ".")
+        git("commit", "-m", "hello world")
+
+    copy(
+        str(repo),
+        dst,
+        defaults=True,
+        overwrite=True,
+        vcs_ref="HEAD",
+    )
+
+    assert os.path.exists(dst / "target.txt")
+    assert os.path.exists(dst / "symlink.txt")
+    assert not os.path.islink(dst / "symlink.txt")
+
+
+def test_update_symlink(tmp_path_factory):
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+
+    build_file_tree(
+        {
+            src
+            / ".copier-answers.yml.jinja": """\
+                # Changes here will be overwritten by Copier
+                {{ _copier_answers|to_nice_yaml }}
+            """,
+            src
+            / "copier.yml": """\
+                _preserve_symlinks: true
+            """,
+            src
+            / "aaaa.txt": """
+                Lorem ipsum
+            """,
+            src
+            / "bbbb.txt": """
+                dolor sit amet
+            """,
+            src / "symlink.txt": Path("./aaaa.txt"),
+        }
+    )
+
+    with local.cwd(src):
+        git("init")
+        git("add", "-A")
+        git("commit", "-m", "first commit on src")
+
+    run_copy(str(src), dst, defaults=True, overwrite=True)
+
+    with local.cwd(src):
+        # test updating a symlink
+        os.remove("symlink.txt")
+        os.symlink("bbbb.txt", "symlink.txt")
+
+    # dst must be vcs-tracked to use run_update
+    with local.cwd(dst):
+        git("init")
+        git("add", "-A")
+        git("commit", "-m", "first commit on dst")
+
+    # make sure changes have not yet propagated
+    p1 = src / "symlink.txt"
+    p2 = dst / "symlink.txt"
+    assert p1.read_text() != p2.read_text()
+
+    with pytest.warns(DirtyLocalWarning):
+        run_update(dst, defaults=True, overwrite=True)
+
+    # make sure changes propagate after update
+    p1 = src / "symlink.txt"
+    p2 = dst / "symlink.txt"
+    assert p1.read_text() == p2.read_text()
+
+    assert readlink(dst / "symlink.txt") == Path("bbbb.txt")
+
+
+def test_exclude_symlink(tmp_path_factory):
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    # Prepare repo bundle
+    repo = src / "repo"
+    repo.mkdir()
+    build_file_tree(
+        {
+            repo
+            / "copier.yaml": """\
+            _preserve_symlinks: true
+            """,
+            repo / "target.txt": "Symlink target",
+            repo / "symlink.txt": Path("target.txt"),
+        }
+    )
+
+    with local.cwd(src):
+        git("init")
+        git("add", ".")
+        git("commit", "-m", "hello world")
+
+    copy(
+        str(repo),
+        dst,
+        defaults=True,
+        overwrite=True,
+        exclude=["symlink.txt"],
+        vcs_ref="HEAD",
+    )
+    assert not (dst / "symlink.txt").exists()
+    assert not (dst / "symlink.txt").is_symlink()
+
+
+def test_pretend_symlink(tmp_path_factory):
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    # Prepare repo bundle
+    repo = src / "repo"
+    repo.mkdir()
+    build_file_tree(
+        {
+            repo
+            / "copier.yaml": """\
+            _preserve_symlinks: true
+            """,
+            repo / "target.txt": "Symlink target",
+            repo / "symlink.txt": Path("target.txt"),
+        }
+    )
+
+    with local.cwd(src):
+        git("init")
+        git("add", ".")
+        git("commit", "-m", "hello world")
+
+    copy(
+        str(repo),
+        dst,
+        defaults=True,
+        overwrite=True,
+        pretend=True,
+        vcs_ref="HEAD",
+    )
+    assert not (dst / "symlink.txt").exists()
+    assert not (dst / "symlink.txt").is_symlink()
+
+
+def test_copy_symlink_none_path(tmp_path_factory):
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    # Prepare repo bundle
+    repo = src / "repo"
+    repo.mkdir()
+    build_file_tree(
+        {
+            repo
+            / "copier.yaml": """\
+            _preserve_symlinks: true
+            render: false
+            """,
+            repo / "target.txt": "Symlink target",
+            repo / "{% if render %}symlink.txt{% endif %}": Path("target.txt"),
+        }
+    )
+
+    with local.cwd(src):
+        git("init")
+        git("add", ".")
+        git("commit", "-m", "hello world")
+
+    copy(
+        str(repo),
+        dst,
+        defaults=True,
+        overwrite=True,
+        vcs_ref="HEAD",
+    )
+
+    assert os.path.exists(dst / "target.txt")
+    assert not os.path.exists(dst / "symlink.txt")
+    assert not os.path.islink(dst / "symlink.txt")