feat: support preserving symlinks when copying templates (#938)

* feat: Preserve symlinks when copying templates

* test: Add tests for symlink copying
This commit is contained in:
Adrian Freund 2023-04-07 09:03:17 +02:00 committed by GitHub
parent c96ff3cf5e
commit 0f610be801
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 524 additions and 17 deletions

View File

@ -11,7 +11,7 @@ from functools import partial
from itertools import chain
from pathlib import Path
from shutil import rmtree
from typing import Callable, Iterable, Mapping, Optional, Sequence
from typing import Callable, Iterable, Mapping, Optional, Sequence, Union
from unicodedata import normalize
from jinja2.loaders import FileSystemLoader
@ -29,7 +29,7 @@ from questionary import unsafe_prompt
from .errors import CopierAnswersInterrupt, ExtensionNotFoundError, UserMessageError
from .subproject import Subproject
from .template import Task, Template
from .tools import Style, TemporaryDirectory, printf
from .tools import Style, TemporaryDirectory, printf, readlink
from .types import (
AnyByStrDict,
JSONSerializable,
@ -302,7 +302,11 @@ class Worker:
return bool(ask(f" Overwrite {dst_relpath}?", default=True))
def _render_allowed(
self, dst_relpath: Path, is_dir: bool = False, expected_contents: bytes = b""
self,
dst_relpath: Path,
is_dir: bool = False,
is_symlink: bool = False,
expected_contents: Union[bytes, Path] = b"",
) -> bool:
"""Determine if a file or directory can be rendered.
@ -311,6 +315,8 @@ class Worker:
Relative path to destination.
is_dir:
Indicate if the path must be treated as a directory or not.
is_symlink:
Indicate if the path must be treated as a symlink or not.
expected_contents:
Used to compare existing file contents with them. Allows to know if
rendering is needed.
@ -321,7 +327,11 @@ class Worker:
if dst_relpath != Path(".") and self.match_exclude(dst_relpath):
return False
try:
previous_content = dst_abspath.read_bytes()
previous_content: Union[bytes, Path]
if is_symlink:
previous_content = readlink(dst_abspath)
else:
previous_content = dst_abspath.read_bytes()
except FileNotFoundError:
printf(
"create",
@ -519,6 +529,45 @@ class Worker:
dst_abspath.write_bytes(new_content)
dst_abspath.chmod(src_mode)
def _render_symlink(self, src_abspath: Path) -> None:
"""Render one symlink.
Args:
src_abspath:
Symlink to be rendered. It must be an absolute path within
the template.
"""
assert src_abspath.is_absolute()
src_relpath = src_abspath.relative_to(self.template_copy_root)
dst_relpath = self._render_path(src_relpath)
if dst_relpath is None:
return
dst_abspath = Path(self.subproject.local_abspath, dst_relpath)
src_target = readlink(src_abspath)
if src_abspath.name.endswith(self.template.templates_suffix):
dst_target = Path(self._render_string(str(src_target)))
else:
dst_target = src_target
if not self._render_allowed(
dst_relpath,
expected_contents=dst_target,
is_symlink=True,
):
return
if not self.pretend:
# symlink_to doesn't overwrite existing files, so delete it first
if dst_abspath.is_symlink() or dst_abspath.exists():
dst_abspath.unlink()
dst_abspath.symlink_to(dst_target)
if sys.platform == "darwin":
# Only macOS supports permissions on symlinks.
# Other platforms just copy the permission of the target
src_mode = src_abspath.lstat().st_mode
dst_abspath.lchmod(src_mode)
def _render_folder(self, src_abspath: Path) -> None:
"""Recursively render a folder.
@ -540,6 +589,8 @@ class Worker:
for file in src_abspath.iterdir():
if file.is_dir():
self._render_folder(file)
elif file.is_symlink() and self.template.preserve_symlinks:
self._render_symlink(file)
else:
self._render_file(file)

View File

@ -446,6 +446,14 @@ class Template:
return DEFAULT_TEMPLATES_SUFFIX
return result
@cached_property
def preserve_symlinks(self) -> bool:
"""Know if Copier should preserve symlinks when rendering the template.
See [preserve_symlinks][].
"""
return bool(self.config_data.get("preserve_symlinks", False))
@cached_property
def local_abspath(self) -> Path:
"""Get the absolute path to the template on disk.

View File

@ -178,3 +178,17 @@ class TemporaryDirectory(tempfile.TemporaryDirectory):
@staticmethod
def _robust_cleanup(name):
shutil.rmtree(name, ignore_errors=False, onerror=handle_remove_readonly)
def readlink(link: Path) -> Path:
"""A custom version of os.readlink/pathlib.Path.readlink.
pathlib.Path.readlink is what we ideally would want to use, but it is only available on python>=3.9.
os.readlink doesn't support Path and bytes on Windows for python<3.8
"""
if sys.version_info >= (3, 9):
return link.readlink()
elif sys.version_info >= (3, 8) or os.name != "nt":
return Path(os.readlink(link))
else:
return Path(os.readlink(str(link)))

View File

@ -1022,6 +1022,18 @@ Run but do not make any changes.
Not supported in `copier.yml`.
### `preserve_symlinks`
- Format: `bool`
- CLI flags: N/A
- Default value: `False`
Keep symlinks as symlinks. If this is set to `False` symlinks will be replaced with the
file they point to.
When set to `True` and the symlink ends with the template suffix (`.jinja` by default)
the target path of the symlink will be rendered as a jinja template.
### `quiet`
- Format: `bool`

1
tests/demo/symlink.txt Symbolic link
View File

@ -0,0 +1 @@
aaaa.txt

View File

@ -6,7 +6,7 @@ import textwrap
from enum import Enum
from hashlib import sha1
from pathlib import Path
from typing import Mapping, Optional, Tuple, Union, cast
from typing import Mapping, Optional, Tuple, Union
from pexpect.popen_spawn import PopenSpawn
from plumbum import local
@ -100,18 +100,30 @@ def assert_file(tmp_path: Path, *path: str) -> None:
def build_file_tree(
spec: Mapping[StrOrPath, Union[str, bytes]], dedent: bool = True
) -> None:
"""Builds a file tree based on the received spec."""
spec: Mapping[StrOrPath, Union[str, bytes, Path]], dedent: bool = True
):
"""Builds a file tree based on the received spec.
Params:
spec:
A mapping from filesystem paths to file contents. If the content is
a Path object a symlink to the path will be created instead.
dedent: Dedent file contents.
"""
for path, contents in spec.items():
path = Path(path)
binary = isinstance(contents, bytes)
if not binary and dedent:
contents = textwrap.dedent(cast(str, contents))
path.parent.mkdir(parents=True, exist_ok=True)
mode = "wb" if binary else "w"
with path.open(mode) as fd:
fd.write(contents)
if isinstance(contents, Path):
os.symlink(str(contents), path)
else:
binary = isinstance(contents, bytes)
if not binary and dedent:
assert isinstance(contents, str)
contents = textwrap.dedent(contents)
mode = "wb" if binary else "w"
with path.open(mode) as fd:
fd.write(contents)
def expect_prompt(

View File

@ -135,6 +135,8 @@ def test_copy(tmp_path: Path) -> None:
assert not (tmp_path / "py2_folder" / "thing.py").exists()
assert (tmp_path / "py3_folder" / "thing.py").exists()
assert (tmp_path / "aaaa.txt").exists()
@pytest.mark.impure
def test_copy_repo(tmp_path: Path) -> None:
@ -288,22 +290,28 @@ def test_empty_dir(tmp_path_factory: pytest.TempPathFactory, generate: bool) ->
),
(src / "tpl" / "two.txt"): "[[ do_it ]]",
(src / "tpl" / "[% if do_it %]three.txt[% endif %].jinja"): "[[ do_it ]]",
(src / "tpl" / "four" / "[% if do_it %]five.txt[% endif %].jinja"): (
(src / "tpl" / "[% if do_it %]four.txt[% endif %].jinja"): Path(
"[% if do_it %]three.txt[% endif %].jinja"
),
(src / "tpl" / "five" / "[% if do_it %]six.txt[% endif %].jinja"): (
"[[ do_it ]]"
),
},
)
copier.run_copy(str(src), dst, {"do_it": generate}, defaults=True, overwrite=True)
assert (dst / "four").is_dir()
assert (dst / "five").is_dir()
assert (dst / "two.txt").read_text() == "[[ do_it ]]"
assert (dst / "one_dir").exists() == generate
assert (dst / "three.txt").exists() == generate
assert (dst / "four.txt").exists() == generate
assert (dst / "one_dir").is_dir() == generate
assert (dst / "one_dir" / "one.txt").is_file() == generate
if generate:
assert (dst / "one_dir" / "one.txt").read_text() == repr(generate)
assert (dst / "three.txt").read_text() == repr(generate)
assert (dst / "four" / "five.txt").read_text() == repr(generate)
assert not (dst / "four.txt").is_symlink()
assert (dst / "four.txt").read_text() == repr(generate)
assert (dst / "five" / "six.txt").read_text() == repr(generate)
@pytest.mark.skipif(

View File

@ -66,6 +66,7 @@ def test_update(tmp_path_factory: pytest.TempPathFactory) -> None:
delete me.
"""
),
(src / "symlink.txt"): Path("./to_delete.txt"),
}
)
@ -84,6 +85,10 @@ def test_update(tmp_path_factory: pytest.TempPathFactory) -> None:
with open("aaaa.txt", "a") as f:
f.write("dolor sit amet")
# test updating a symlink
Path("symlink.txt").unlink()
Path("symlink.txt").symlink_to("test_file.txt")
# test removing a file
Path("to_delete.txt").unlink()
@ -98,6 +103,10 @@ def test_update(tmp_path_factory: pytest.TempPathFactory) -> None:
assert (src / "aaaa.txt").read_text() != (dst / "aaaa.txt").read_text()
p1 = src / "symlink.txt"
p2 = dst / "symlink.txt"
assert p1.read_text() != p2.read_text()
assert (dst / "to_delete.txt").exists()
with pytest.warns(DirtyLocalWarning):
@ -108,6 +117,11 @@ def test_update(tmp_path_factory: pytest.TempPathFactory) -> None:
assert (src / "aaaa.txt").read_text() == (dst / "aaaa.txt").read_text()
p1 = src / "symlink.txt"
p2 = dst / "symlink.txt"
assert p1.read_text() == p2.read_text()
assert not (dst / "symlink.txt").is_symlink()
# HACK https://github.com/copier-org/copier/issues/461
# TODO test file deletion on update
# assert not (dst / "to_delete.txt").exists()

387
tests/test_symlinks.py Normal file
View File

@ -0,0 +1,387 @@
import os
from pathlib import Path
import pytest
from plumbum import local
from plumbum.cmd import git
from copier import copy, readlink, run_copy, run_update
from copier.errors import DirtyLocalWarning
from .helpers import build_file_tree
def test_copy_symlink(tmp_path_factory):
src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
# Prepare repo bundle
repo = src / "repo"
repo.mkdir()
build_file_tree(
{
repo
/ "copier.yaml": """\
_preserve_symlinks: true
""",
repo / "target.txt": "Symlink target",
repo / "symlink.txt": Path("target.txt"),
}
)
with local.cwd(src):
git("init")
git("add", ".")
git("commit", "-m", "hello world")
copy(
str(repo),
dst,
defaults=True,
overwrite=True,
vcs_ref="HEAD",
)
assert os.path.exists(dst / "target.txt")
assert os.path.exists(dst / "symlink.txt")
assert os.path.islink(dst / "symlink.txt")
assert readlink(dst / "symlink.txt") == Path("target.txt")
def test_copy_symlink_templated_name(tmp_path_factory):
src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
# Prepare repo bundle
repo = src / "repo"
repo.mkdir()
build_file_tree(
{
repo
/ "copier.yaml": """\
_preserve_symlinks: true
symlink_name: symlink
""",
repo / "target.txt": "Symlink target",
repo / "{{ symlink_name }}.txt": Path("target.txt"),
}
)
with local.cwd(src):
git("init")
git("add", ".")
git("commit", "-m", "hello world")
copy(
str(repo),
dst,
defaults=True,
overwrite=True,
vcs_ref="HEAD",
)
assert os.path.exists(dst / "target.txt")
assert os.path.exists(dst / "symlink.txt")
assert os.path.islink(dst / "symlink.txt")
assert readlink(dst / "symlink.txt") == Path("target.txt")
def test_copy_symlink_templated_target(tmp_path_factory):
src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
# Prepare repo bundle
repo = src / "repo"
repo.mkdir()
build_file_tree(
{
repo
/ "copier.yaml": """\
_preserve_symlinks: true
target_name: target
""",
repo / "{{ target_name }}.txt": "Symlink target",
repo / "symlink1.txt.jinja": Path("{{ target_name }}.txt"),
repo / "symlink2.txt": Path("{{ target_name }}.txt"),
}
)
with local.cwd(src):
git("init")
git("add", ".")
git("commit", "-m", "hello world")
copy(
str(repo),
dst,
defaults=True,
overwrite=True,
vcs_ref="HEAD",
)
assert os.path.exists(dst / "target.txt")
assert os.path.exists(dst / "symlink1.txt")
assert os.path.islink(dst / "symlink1.txt")
assert readlink(dst / "symlink1.txt") == Path("target.txt")
assert not os.path.exists(dst / "symlink2.txt")
assert os.path.islink(dst / "symlink2.txt")
assert readlink(dst / "symlink2.txt") == Path("{{ target_name }}.txt")
def test_copy_symlink_missing_target(tmp_path_factory):
src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
# Prepare repo bundle
repo = src / "repo"
repo.mkdir()
build_file_tree(
{
repo
/ "copier.yaml": """\
_preserve_symlinks: true
""",
repo / "symlink.txt": Path("target.txt"),
}
)
with local.cwd(src):
git("init")
git("add", ".")
git("commit", "-m", "hello world")
copy(
str(repo),
dst,
defaults=True,
overwrite=True,
vcs_ref="HEAD",
)
assert os.path.islink(dst / "symlink.txt")
assert readlink(dst / "symlink.txt") == Path("target.txt")
assert not os.path.exists(
dst / "symlink.txt"
) # exists follows symlinks, It returns False as the target doesn't exist
def test_option_preserve_symlinks_false(tmp_path_factory):
src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
# Prepare repo bundle
repo = src / "repo"
repo.mkdir()
build_file_tree(
{
repo
/ "copier.yaml": """\
_preserve_symlinks: false
""",
repo / "target.txt": "Symlink target",
repo / "symlink.txt": Path("target.txt"),
}
)
with local.cwd(src):
git("init")
git("add", ".")
git("commit", "-m", "hello world")
copy(
str(repo),
dst,
defaults=True,
overwrite=True,
vcs_ref="HEAD",
)
assert os.path.exists(dst / "target.txt")
assert os.path.exists(dst / "symlink.txt")
assert not os.path.islink(dst / "symlink.txt")
def test_option_preserve_symlinks_default(tmp_path_factory):
src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
# Prepare repo bundle
repo = src / "repo"
repo.mkdir()
build_file_tree(
{
repo
/ "copier.yaml": """\
""",
repo / "target.txt": "Symlink target",
repo / "symlink.txt": Path("target.txt"),
}
)
with local.cwd(src):
git("init")
git("add", ".")
git("commit", "-m", "hello world")
copy(
str(repo),
dst,
defaults=True,
overwrite=True,
vcs_ref="HEAD",
)
assert os.path.exists(dst / "target.txt")
assert os.path.exists(dst / "symlink.txt")
assert not os.path.islink(dst / "symlink.txt")
def test_update_symlink(tmp_path_factory):
src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
build_file_tree(
{
src
/ ".copier-answers.yml.jinja": """\
# Changes here will be overwritten by Copier
{{ _copier_answers|to_nice_yaml }}
""",
src
/ "copier.yml": """\
_preserve_symlinks: true
""",
src
/ "aaaa.txt": """
Lorem ipsum
""",
src
/ "bbbb.txt": """
dolor sit amet
""",
src / "symlink.txt": Path("./aaaa.txt"),
}
)
with local.cwd(src):
git("init")
git("add", "-A")
git("commit", "-m", "first commit on src")
run_copy(str(src), dst, defaults=True, overwrite=True)
with local.cwd(src):
# test updating a symlink
os.remove("symlink.txt")
os.symlink("bbbb.txt", "symlink.txt")
# dst must be vcs-tracked to use run_update
with local.cwd(dst):
git("init")
git("add", "-A")
git("commit", "-m", "first commit on dst")
# make sure changes have not yet propagated
p1 = src / "symlink.txt"
p2 = dst / "symlink.txt"
assert p1.read_text() != p2.read_text()
with pytest.warns(DirtyLocalWarning):
run_update(dst, defaults=True, overwrite=True)
# make sure changes propagate after update
p1 = src / "symlink.txt"
p2 = dst / "symlink.txt"
assert p1.read_text() == p2.read_text()
assert readlink(dst / "symlink.txt") == Path("bbbb.txt")
def test_exclude_symlink(tmp_path_factory):
src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
# Prepare repo bundle
repo = src / "repo"
repo.mkdir()
build_file_tree(
{
repo
/ "copier.yaml": """\
_preserve_symlinks: true
""",
repo / "target.txt": "Symlink target",
repo / "symlink.txt": Path("target.txt"),
}
)
with local.cwd(src):
git("init")
git("add", ".")
git("commit", "-m", "hello world")
copy(
str(repo),
dst,
defaults=True,
overwrite=True,
exclude=["symlink.txt"],
vcs_ref="HEAD",
)
assert not (dst / "symlink.txt").exists()
assert not (dst / "symlink.txt").is_symlink()
def test_pretend_symlink(tmp_path_factory):
src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
# Prepare repo bundle
repo = src / "repo"
repo.mkdir()
build_file_tree(
{
repo
/ "copier.yaml": """\
_preserve_symlinks: true
""",
repo / "target.txt": "Symlink target",
repo / "symlink.txt": Path("target.txt"),
}
)
with local.cwd(src):
git("init")
git("add", ".")
git("commit", "-m", "hello world")
copy(
str(repo),
dst,
defaults=True,
overwrite=True,
pretend=True,
vcs_ref="HEAD",
)
assert not (dst / "symlink.txt").exists()
assert not (dst / "symlink.txt").is_symlink()
def test_copy_symlink_none_path(tmp_path_factory):
src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
# Prepare repo bundle
repo = src / "repo"
repo.mkdir()
build_file_tree(
{
repo
/ "copier.yaml": """\
_preserve_symlinks: true
render: false
""",
repo / "target.txt": "Symlink target",
repo / "{% if render %}symlink.txt{% endif %}": Path("target.txt"),
}
)
with local.cwd(src):
git("init")
git("add", ".")
git("commit", "-m", "hello world")
copy(
str(repo),
dst,
defaults=True,
overwrite=True,
vcs_ref="HEAD",
)
assert os.path.exists(dst / "target.txt")
assert not os.path.exists(dst / "symlink.txt")
assert not os.path.islink(dst / "symlink.txt")