feat: add dynamic file structures in loop using yield-tag (#1855)

Add jinja2 extension for yield tag, allow _render_path to generate multiple paths and contexts when yield tag is used.

The tag is only allowed in path render contexts. When rendering within a file, it doesn't make sense. Use the normal for tag there. If you use yield, you'll get an exception.

Fixes https://github.com/copier-org/copier/issues/1271

copier/errors.py

@@ -112,6 +112,14 @@ class UnsafeTemplateError(CopierError):
         )
 
 
+class YieldTagInFileError(CopierError):
+    """A yield tag is used in the file content, but it is not allowed."""
+
+
+class MultipleYieldTagsError(CopierError):
+    """Multiple yield tags are used in one path name, but it is not allowed."""
+
+
 # Warnings
 class CopierWarning(Warning):
     """Base class for all other Copier warnings."""

copier/jinja_ext.py

@@ -0,0 +1,123 @@
+"""Jinja2 extensions built for Copier."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Iterable
+
+from jinja2 import nodes
+from jinja2.exceptions import UndefinedError
+from jinja2.ext import Extension
+from jinja2.parser import Parser
+from jinja2.sandbox import SandboxedEnvironment
+
+from copier.errors import MultipleYieldTagsError
+
+
+class YieldEnvironment(SandboxedEnvironment):
+    """Jinja2 environment with attributes from the YieldExtension.
+
+    This is simple environment class that extends the SandboxedEnvironment
+    for use with the YieldExtension, mainly for avoiding type errors.
+
+    We use the SandboxedEnvironment because we want to minimize the risk of hidden malware
+    in the templates. Of course we still have the post-copy tasks to worry about, but at least
+    they are more visible to the final user.
+    """
+
+    yield_name: str | None
+    yield_iterable: Iterable[Any] | None
+
+    def __init__(self, *args: Any, **kwargs: Any):
+        super().__init__(*args, **kwargs)
+        self.extend(yield_name=None, yield_iterable=None)
+
+
+class YieldExtension(Extension):
+    """Jinja2 extension for the `yield` tag.
+
+    If `yield` tag is used in a template, this extension sets following attribute to the
+    jinja environment:
+
+    - `yield_name`: The name of the variable that will be yielded.
+    - `yield_iterable`: The variable that will be looped over.
+
+    Note that this extension just sets the attributes but renders templates as usual.
+    It is the caller's responsibility to use the `yield_context` attribute in the template to
+    generate the desired output.
+
+    !!! example
+
+        ```pycon
+        >>> from copier.jinja_ext import YieldEnvironment, YieldExtension
+        >>> env = YieldEnvironment(extensions=[YieldExtension])
+        >>> template = env.from_string("{% yield single_var from looped_var %}{{ single_var }}{% endyield %}")
+        >>> template.render({"looped_var": [1, 2, 3]})
+        ''
+        >>> env.yield_name
+        'single_var'
+        >>> env.yield_iterable
+        [1, 2, 3]
+        ```
+    """
+
+    tags = {"yield"}
+
+    environment: YieldEnvironment
+
+    def preprocess(
+        self, source: str, _name: str | None, _filename: str | None = None
+    ) -> str:
+        """Preprocess hook to reset attributes before rendering."""
+        self.environment.yield_name = self.environment.yield_iterable = None
+
+        return source
+
+    def parse(self, parser: Parser) -> nodes.Node:
+        """Parse the `yield` tag."""
+        lineno = next(parser.stream).lineno
+
+        yield_name: nodes.Name = parser.parse_assign_target(name_only=True)
+        parser.stream.expect("name:from")
+        yield_iterable = parser.parse_expression()
+        body = parser.parse_statements(("name:endyield",), drop_needle=True)
+
+        return nodes.CallBlock(
+            self.call_method(
+                "_yield_support",
+                [nodes.Const(yield_name.name), yield_iterable],
+            ),
+            [],
+            [],
+            body,
+            lineno=lineno,
+        )
+
+    def _yield_support(
+        self, yield_name: str, yield_iterable: Iterable[Any], caller: Callable[[], str]
+    ) -> str:
+        """Support function for the yield tag.
+
+        Sets the `yield_name` and `yield_iterable` attributes in the environment then calls
+        the provided caller function. If an UndefinedError is raised, it returns an empty string.
+        """
+        if (
+            self.environment.yield_name is not None
+            or self.environment.yield_iterable is not None
+        ):
+            raise MultipleYieldTagsError(
+                "Attempted to parse the yield tag twice. Only one yield tag is allowed per path name.\n"
+                f'A yield tag with the name: "{self.environment.yield_name}" and iterable: "{self.environment.yield_iterable}" already exists.'
+            )
+
+        self.environment.yield_name = yield_name
+        self.environment.yield_iterable = yield_iterable
+
+        try:
+            res = caller()
+
+        # expression like `dict.attr` will always raise UndefinedError
+        # so we catch it here and return an empty string
+        except UndefinedError:
+            res = ""
+
+        return res

copier/main.py

@@ -29,7 +29,6 @@ from typing import (
 from unicodedata import normalize
 
 from jinja2.loaders import FileSystemLoader
-from jinja2.sandbox import SandboxedEnvironment
 from pathspec import PathSpec
 from plumbum import ProcessExecutionError, colors
 from plumbum.cli.terminal import ask
@@ -44,7 +43,9 @@ from .errors import (
     ExtensionNotFoundError,
     UnsafeTemplateError,
     UserMessageError,
+    YieldTagInFileError,
 )
+from .jinja_ext import YieldEnvironment, YieldExtension
 from .subproject import Subproject
 from .template import Task, Template
 from .tools import (
@@ -541,7 +542,7 @@ class Worker:
         return self.template.exclude + tuple(self.exclude)
 
     @cached_property
-    def jinja_env(self) -> SandboxedEnvironment:
+    def jinja_env(self) -> YieldEnvironment:
         """Return a pre-configured Jinja environment.
 
         Respects template settings.
@@ -550,14 +551,11 @@ class Worker:
         loader = FileSystemLoader(paths)
         default_extensions = [
             "jinja2_ansible_filters.AnsibleCoreFiltersExtension",
+            YieldExtension,
         ]
         extensions = default_extensions + list(self.template.jinja_extensions)
-        # We want to minimize the risk of hidden malware in the templates
-        # so we use the SandboxedEnvironment instead of the regular one.
-        # Of course we still have the post-copy tasks to worry about, but at least
-        # they are more visible to the final user.
         try:
-            env = SandboxedEnvironment(
+            env = YieldEnvironment(
                 loader=loader, extensions=extensions, **self.template.envops
             )
         except ModuleNotFoundError as error:
@@ -607,19 +605,25 @@ class Worker:
         for src in scantree(str(self.template_copy_root), follow_symlinks):
             src_abspath = Path(src.path)
             src_relpath = Path(src_abspath).relative_to(self.template.local_abspath)
-            dst_relpath = self._render_path(
+            dst_relpaths_ctxs = self._render_path(
                 Path(src_abspath).relative_to(self.template_copy_root)
             )
-            if dst_relpath is None or self.match_exclude(dst_relpath):
-                continue
-            if src.is_symlink() and self.template.preserve_symlinks:
-                self._render_symlink(src_relpath, dst_relpath)
-            elif src.is_dir(follow_symlinks=follow_symlinks):
-                self._render_folder(dst_relpath)
-            else:
-                self._render_file(src_relpath, dst_relpath)
+            for dst_relpath, ctx in dst_relpaths_ctxs:
+                if self.match_exclude(dst_relpath):
+                    continue
+                if src.is_symlink() and self.template.preserve_symlinks:
+                    self._render_symlink(src_relpath, dst_relpath)
+                elif src.is_dir(follow_symlinks=follow_symlinks):
+                    self._render_folder(dst_relpath)
+                else:
+                    self._render_file(src_relpath, dst_relpath, extra_context=ctx or {})
 
-    def _render_file(self, src_relpath: Path, dst_relpath: Path) -> None:
+    def _render_file(
+        self,
+        src_relpath: Path,
+        dst_relpath: Path,
+        extra_context: AnyByStrDict | None = None,
+    ) -> None:
         """Render one file.
 
         Args:
@@ -629,6 +633,8 @@ class Worker:
             dst_relpath:
                 File to be created. It must be a path relative to the subproject
                 root.
+            extra_context:
+                Additional variables to use for rendering the template.
         """
         # TODO Get from main.render_file()
         assert not src_relpath.is_absolute()
@@ -644,7 +650,13 @@ class Worker:
                 # suffix is empty, fallback to copy
                 new_content = src_abspath.read_bytes()
             else:
-                new_content = tpl.render(**self._render_context()).encode()
+                new_content = tpl.render(
+                    **self._render_context(), **(extra_context or {})
+                ).encode()
+                if self.jinja_env.yield_name:
+                    raise YieldTagInFileError(
+                        f"File {src_relpath} contains a yield tag, but it is not allowed."
+                    )
         else:
             new_content = src_abspath.read_bytes()
         dst_abspath = self.subproject.local_abspath / dst_relpath
@@ -716,8 +728,85 @@ class Worker:
             dst_abspath = self.subproject.local_abspath / dst_relpath
             dst_abspath.mkdir(parents=True, exist_ok=True)
 
-    def _render_path(self, relpath: Path) -> Path | None:
-        """Render one relative path.
+    def _adjust_rendered_part(self, rendered_part: str) -> str:
+        """Adjust the rendered part if necessary.
+
+        If `{{ _copier_conf.answers_file }}` becomes the full path,
+        restore part to be just the end leaf.
+
+        Args:
+            rendered_part:
+                The rendered part of the path to adjust.
+
+        """
+        if str(self.answers_relpath) == rendered_part:
+            return Path(rendered_part).name
+        return rendered_part
+
+    def _render_parts(
+        self,
+        parts: tuple[str, ...],
+        rendered_parts: tuple[str, ...] | None = None,
+        extra_context: AnyByStrDict | None = None,
+        is_template: bool = False,
+    ) -> Iterable[tuple[Path, AnyByStrDict | None]]:
+        """Render a set of parts into path and context pairs.
+
+        If a yield tag is found in a part, it will recursively yield multiple path and context pairs.
+        """
+        if rendered_parts is None:
+            rendered_parts = tuple()
+
+        if not parts:
+            rendered_path = Path(*rendered_parts)
+
+            templated_sibling = (
+                self.template.local_abspath
+                / f"{rendered_path}{self.template.templates_suffix}"
+            )
+            if is_template or not templated_sibling.exists():
+                yield rendered_path, extra_context
+
+            return
+
+        part = parts[0]
+        parts = parts[1:]
+
+        if not extra_context:
+            extra_context = {}
+
+        # If the `part` has a yield tag, `self.jinja_env` will be set with the yield name and iterable
+        rendered_part = self._render_string(part, extra_context=extra_context)
+
+        yield_name = self.jinja_env.yield_name
+        if yield_name:
+            for value in self.jinja_env.yield_iterable or ():
+                new_context = {**extra_context, yield_name: value}
+                rendered_part = self._render_string(part, extra_context=new_context)
+                rendered_part = self._adjust_rendered_part(rendered_part)
+
+                # Skip if any part is rendered as an empty string
+                if not rendered_part:
+                    continue
+
+                yield from self._render_parts(
+                    parts, rendered_parts + (rendered_part,), new_context, is_template
+                )
+
+            return
+
+        # Skip if any part is rendered as an empty string
+        if not rendered_part:
+            return
+
+        rendered_part = self._adjust_rendered_part(rendered_part)
+
+        yield from self._render_parts(
+            parts, rendered_parts + (rendered_part,), extra_context, is_template
+        )
+
+    def _render_path(self, relpath: Path) -> Iterable[tuple[Path, AnyByStrDict | None]]:
+        """Render one relative path into multiple path and context pairs.
 
         Args:
             relpath:
@@ -729,29 +818,11 @@ class Worker:
         )
         # With an empty suffix, the templated sibling always exists.
         if templated_sibling.exists() and self.template.templates_suffix:
-            return None
+            return
         if self.template.templates_suffix and is_template:
             relpath = relpath.with_suffix("")
-        rendered_parts = []
-        for part in relpath.parts:
-            # Skip folder if any part is rendered as an empty string
-            part = self._render_string(part)
-            if not part:
-                return None
-            # {{ _copier_conf.answers_file }} becomes the full path; in that case,
-            # restore part to be just the end leaf
-            if str(self.answers_relpath) == part:
-                part = Path(part).name
-            rendered_parts.append(part)
-        result = Path(*rendered_parts)
-        if not is_template:
-            templated_sibling = (
-                self.template.local_abspath
-                / f"{result}{self.template.templates_suffix}"
-            )
-            if templated_sibling.exists():
-                return None
-        return result
+
+        yield from self._render_parts(relpath.parts, is_template=is_template)
 
     def _render_string(
         self, string: str, extra_context: AnyByStrDict | None = None

docs/comparisons.md

@@ -18,6 +18,7 @@ docs! We don't want to be biased, but it's easy that we tend to be:
 | Feature                                  | Copier                           | Cookiecutter                    | Yeoman        |
 | ---------------------------------------- | -------------------------------- | ------------------------------- | ------------- |
 | Can template file names                  | Yes                              | Yes                             | Yes           |
+| Can generate file structures in loops    | Yes                              | No                              | No            |
 | Configuration                            | Single YAML file[^1]             | Single JSON file                | JS module     |
 | Migrations                               | Yes                              | No                              | No            |
 | Programmed in                            | Python                           | Python                          | NodeJS        |

docs/creating.md

@@ -130,3 +130,75 @@ The name of the project root directory.
 Some rendering contexts provide variables unique to them:
 
 -   [`migrations`](configuring.md#migrations)
+
+## Loop over lists to generate files and directories
+
+You can use the special `yield` tag in file and directory names to generate multiple
+files or directories based on a list of items.
+
+In the path name, `{% yield item from list_of_items %}{{ item }}{% endyield %}` will
+loop over the `list_of_items` and replace `{{ item }}` with each item in the list.
+
+A looped `{{ item }}` will be available in the scope of generated files and directories.
+
+```yaml title="copier.yml"
+commands:
+    type: yaml
+    multiselect: true
+    choices:
+        init:
+            value: &init
+                name: init
+                subcommands:
+                    - config
+                    - database
+        run:
+            value: &run
+                name: run
+                subcommands:
+                    - server
+                    - worker
+        deploy:
+            value: &deploy
+                name: deploy
+                subcommands:
+                    - staging
+                    - production
+    default: [*init, *run, *deploy]
+```
+
+```tree result="shell"
+commands
+    {% yield cmd from commands %}{{ cmd.name }}{% endyield %}
+        __init__.py
+        {% yield subcmd from cmd.subcommands %}{{ subcmd }}{% endyield %}.py.jinja
+```
+
+```python+jinja title="{% yield subcmd from cmd.subcommands %}{{ subcmd }}{% endyield %}.py.jinja"
+print("This is the `{{ subcmd }}` subcommand in the `{{ cmd.name }}` command")
+```
+
+If you answer with the default to the question, Copier will generate the following
+structure:
+
+```tree result="shell"
+commands
+    deploy
+        __init__.py
+        production.py
+        staging.py
+    init
+        __init__.py
+        config.py
+        database.py
+    run
+        __init__.py
+        server.py
+        worker.py
+```
+
+Where looped variables `cmd` and `subcmd` are rendered in generated files:
+
+```python title="commands/init/config.py"
+print("This is the `config` subcommand in the `init` command")
+```

docs/reference/jinja_ext.md

@@ -0,0 +1 @@
+::: copier.jinja_ext

mkdocs.yml

@@ -14,6 +14,7 @@ nav:
   - Reference:
       - cli.py: "reference/cli.md"
       - errors.py: "reference/errors.md"
+      - jinja_ext.py: "reference/jinja_ext.md"
       - main.py: "reference/main.md"
       - subproject.py: "reference/subproject.md"
       - template.py: "reference/template.md"

tests/test_dynamic_file_structures.py

@@ -0,0 +1,249 @@
+import warnings
+
+import pytest
+
+import copier
+from copier.errors import MultipleYieldTagsError, YieldTagInFileError
+from tests.helpers import build_file_tree
+
+
+def test_folder_loop(tmp_path_factory: pytest.TempPathFactory) -> None:
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    build_file_tree(
+        {
+            src / "copier.yml": "",
+            src
+            / "folder_loop"
+            / "{% yield item from strings %}{{ item }}{% endyield %}"
+            / "{{ item }}.txt.jinja": "Hello {{ item }}",
+        }
+    )
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        copier.run_copy(
+            str(src),
+            dst,
+            data={
+                "strings": ["a", "b", "c"],
+            },
+            defaults=True,
+            overwrite=True,
+        )
+
+        expected_files = [dst / f"folder_loop/{i}/{i}.txt" for i in ["a", "b", "c"]]
+
+        for f in expected_files:
+            assert f.exists()
+            assert f.read_text() == f"Hello {f.parent.name}"
+
+        all_files = [p for p in dst.rglob("*") if p.is_file()]
+        unexpected_files = set(all_files) - set(expected_files)
+
+        assert not unexpected_files, f"Unexpected files found: {unexpected_files}"
+
+
+def test_nested_folder_loop(tmp_path_factory: pytest.TempPathFactory) -> None:
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    build_file_tree(
+        {
+            src / "copier.yml": "",
+            src
+            / "nested_folder_loop"
+            / "{% yield string_item from strings %}{{ string_item }}{% endyield %}"
+            / "{% yield integer_item from integers %}{{ integer_item }}{% endyield %}"
+            / "{{ string_item }}_{{ integer_item }}.txt.jinja": "Hello {{ string_item }} {{ integer_item }}",
+        }
+    )
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        copier.run_copy(
+            str(src),
+            dst,
+            data={
+                "strings": ["a", "b"],
+                "integers": [1, 2, 3],
+            },
+            defaults=True,
+            overwrite=True,
+        )
+
+        expected_files = [
+            dst / f"nested_folder_loop/{s}/{i}/{s}_{i}.txt"
+            for s in ["a", "b"]
+            for i in [1, 2, 3]
+        ]
+
+        for f in expected_files:
+            assert f.exists()
+            assert f.read_text() == f"Hello {f.parent.parent.name} {f.parent.name}"
+
+        all_files = [p for p in dst.rglob("*") if p.is_file()]
+        unexpected_files = set(all_files) - set(expected_files)
+
+        assert not unexpected_files, f"Unexpected files found: {unexpected_files}"
+
+
+def test_file_loop(tmp_path_factory: pytest.TempPathFactory) -> None:
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    build_file_tree(
+        {
+            src / "copier.yml": "",
+            src
+            / "file_loop"
+            / "{% yield string_item from strings %}{{ string_item }}{% endyield %}.jinja": "Hello {{ string_item }}",
+        }
+    )
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        copier.run_copy(
+            str(src),
+            dst,
+            data={
+                "strings": [
+                    "a.txt",
+                    "b.txt",
+                    "c.txt",
+                    "",
+                ],  # if rendred as '.jinja', it will not be created
+            },
+            defaults=True,
+            overwrite=True,
+        )
+
+        expected_files = [dst / f"file_loop/{i}.txt" for i in ["a", "b", "c"]]
+        for f in expected_files:
+            assert f.exists()
+            assert f.read_text() == f"Hello {f.stem}.txt"
+
+        all_files = [p for p in dst.rglob("*") if p.is_file()]
+        unexpected_files = set(all_files) - set(expected_files)
+
+        assert not unexpected_files, f"Unexpected files found: {unexpected_files}"
+
+
+def test_folder_loop_dict_items(tmp_path_factory: pytest.TempPathFactory) -> None:
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    build_file_tree(
+        {
+            src / "copier.yml": "",
+            src
+            / "folder_loop_dict_items"
+            / "{% yield dict_item from dicts %}{{ dict_item.folder_name }}{% endyield %}"
+            / "{{ dict_item.file_name }}.txt.jinja": "Hello {{ '-'.join(dict_item.content) }}",
+        }
+    )
+
+    dicts = [
+        {
+            "folder_name": "folder_a",
+            "file_name": "file_a",
+            "content": ["folder_a", "file_a"],
+        },
+        {
+            "folder_name": "folder_b",
+            "file_name": "file_b",
+            "content": ["folder_b", "file_b"],
+        },
+        {
+            "folder_name": "folder_c",
+            "file_name": "file_c",
+            "content": ["folder_c", "file_c"],
+        },
+    ]
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+
+        copier.run_copy(
+            str(src),
+            dst,
+            data={"dicts": dicts},
+            defaults=True,
+            overwrite=True,
+        )
+
+        expected_files = [
+            dst / f"folder_loop_dict_items/{d['folder_name']}/{d['file_name']}.txt"
+            for d in [
+                {"folder_name": "folder_a", "file_name": "file_a"},
+                {"folder_name": "folder_b", "file_name": "file_b"},
+                {"folder_name": "folder_c", "file_name": "file_c"},
+            ]
+        ]
+
+        for f in expected_files:
+            assert f.exists()
+            assert f.read_text() == f"Hello {'-'.join([f.parts[-2], f.stem])}"
+
+        all_files = [p for p in dst.rglob("*") if p.is_file()]
+        unexpected_files = set(all_files) - set(expected_files)
+
+        assert not unexpected_files, f"Unexpected files found: {unexpected_files}"
+
+
+def test_raise_yield_tag_in_file(tmp_path_factory: pytest.TempPathFactory) -> None:
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    build_file_tree(
+        {
+            src / "copier.yml": "",
+            src
+            / "file.txt.jinja": "{% yield item from strings %}{{ item }}{% endyield %}",
+        }
+    )
+
+    with pytest.raises(YieldTagInFileError, match="file.txt.jinja"):
+        copier.run_copy(
+            str(src),
+            dst,
+            data={
+                "strings": ["a", "b", "c"],
+            },
+            defaults=True,
+            overwrite=True,
+        )
+
+
+def test_raise_multiple_yield_tags(tmp_path_factory: pytest.TempPathFactory) -> None:
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    # multiple yield tags, not nested
+    file_name = "{% yield item1 from strings %}{{ item1 }}{% endyield %}{% yield item2 from strings %}{{ item2 }}{% endyield %}"
+
+    build_file_tree(
+        {
+            src / "copier.yml": "",
+            src / file_name: "",
+        }
+    )
+
+    with pytest.raises(MultipleYieldTagsError, match="item"):
+        copier.run_copy(
+            str(src),
+            dst,
+            data={
+                "strings": ["a", "b", "c"],
+            },
+            defaults=True,
+            overwrite=True,
+        )
+
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    # multiple yield tags, nested
+    file_name = "{% yield item1 from strings %}{% yield item2 from strings %}{{ item1 }}{{ item2 }}{% endyield %}{% endyield %}"
+
+    build_file_tree(
+        {
+            src / "copier.yml": "",
+            src / file_name: "",
+        }
+    )
+
+    with pytest.raises(MultipleYieldTagsError, match="item"):
+        copier.run_copy(
+            str(src),
+            dst,
+            data={
+                "strings": ["a", "b", "c"],
+            },
+            defaults=True,
+            overwrite=True,
+        )