Add migrations support

This commit fixes #119. Summary:

- Depend on packaging to get a good implementation of PEP 440.
- Document new behaviors.
- Add support for migrations.
- Checkout by default the most modern git tag available in the template.
- Reuse `run_tasks` to run migrations too.
- Use `git describe --tags --always` to obtain template commit, to get a version parseable by PEP 440.
- Update `test_updatediff` to with new behaviors.

@Tecnativa TT20357

README.md

@@ -193,6 +193,18 @@ _tasks:
   - "git init"
   - "rm [[ name_of_the_project ]]/README.md"
 
+# Migrations are like tasks, but they are executed:
+# - Evaluated using PEP 440
+# - In the same order as declared here
+# - Only when new version >= declared version > old version
+# - Only when updating
+_migrations:
+  - version: v1.0.0
+    before:
+      - rm ./old-folder
+    after:
+      - pre-commit install
+
 # Additional paths, from where to search for templates
 _extra_paths:
   - ~/Projects/templates
@@ -232,7 +244,7 @@ should match questions in `copier.yml`.
 The best way to update a project from its template is when all of these conditions are true:
 
 1. The template includes a valid `.copier-answers.yml` file.
-1. The template is versioned with git.
+1. The template is versioned with git (with tags).
 1. The destination folder is versioned with git.
 
 If that's your case, then just enter the destination folder, make sure
@@ -242,6 +254,8 @@ If that's your case, then just enter the destination folder, make sure
 copier update
 ```
 
+This will read all available git tags, will compare them using [PEP 440](https://www.python.org/dev/peps/pep-0440/), and will checkout the latest one before updating. To update to the latest commit, add `--vcs-ref=HEAD`. You can use any other git ref you want.
+
 Copier will do its best to respect the answers you provided when copied for the last
 copy, and the git diff that has evolved since the last copy. If there are conflicts,
 you will probably find diff files around.

copier/cli.py

@@ -67,9 +67,15 @@ class CopierApp(cli.Application):
         ),
     )
     vcs_ref = cli.SwitchAttr(
-        ["-r", "--vcs-ref"], str, help="Git reference to checkout in `template_src`"
+        ["-r", "--vcs-ref"],
+        str,
+        help=(
+            "Git reference to checkout in `template_src`. "
+            "If you do not specify it, it will try to checkout the latest git tag, "
+            "as sorted using the PEP 440 algorithm. If you want to checkout always "
+            "the latest version, use `--vcs-ref=HEAD`.",
+        ),
     )
-
     pretend = cli.Flag(["-n", "--pretend"], help="Run but do not make any changes")
     force = cli.Flag(
         ["-f", "--force"], help="Overwrite files that already exist, without asking"

copier/config/factory.py

@@ -42,7 +42,7 @@ def make_config(
     skip: OptBool = None,
     quiet: OptBool = None,
     cleanup_on_error: OptBool = None,
-    vcs_ref: str = "HEAD",
+    vcs_ref: OptStr = None,
     **kwargs,
 ) -> ConfigData:
     """Provides the configuration object, merged from the different sources.
@@ -71,9 +71,10 @@ def make_config(
     if src_path:
         repo = vcs.get_repo(src_path)
         if repo:
-            src_path = vcs.clone(repo, vcs_ref)
+            src_path = vcs.clone(repo, vcs_ref or "HEAD")
+            vcs_ref = vcs_ref or vcs.checkout_latest_tag(src_path)
             with local.cwd(src_path):
-                _metadata["commit"] = git("rev-parse", "HEAD").strip()
+                _metadata["commit"] = git("describe", "--tags", "--always").strip()
     # Obtain config and query data, asking the user if needed
     file_data = load_config_data(src_path, quiet=True)
     config_data, questions_data = filter_config(file_data)

copier/config/objects.py

@@ -2,7 +2,7 @@ import datetime
 from hashlib import sha512
 from os import urandom
 from pathlib import Path
-from typing import Any, Tuple
+from typing import Any, Sequence, Tuple
 
 from pydantic import BaseModel, Extra, StrictBool, validator
 
@@ -55,15 +55,21 @@ class EnvOps(BaseModel):
         extra = Extra.allow
 
 
+class Migrations(BaseModel):
+    version: str
+    before: StrSeq = ()
+    after: StrSeq = ()
+
+
 class ConfigData(BaseModel):
     src_path: Path
     dst_path: Path
     data: AnyByStrDict = {}
-    extra_paths: PathSeq = []
+    extra_paths: PathSeq = ()
     exclude: StrOrPathSeq = DEFAULT_EXCLUDE
-    include: StrOrPathSeq = []
+    include: StrOrPathSeq = ()
-    skip_if_exists: StrOrPathSeq = []
+    skip_if_exists: StrOrPathSeq = ()
-    tasks: StrSeq = []
+    tasks: StrSeq = ()
     envops: EnvOps = EnvOps()
     templates_suffix: str = DEFAULT_TEMPLATES_SUFFIX
     original_src_path: OptStr
@@ -76,6 +82,7 @@ class ConfigData(BaseModel):
     quiet: StrictBool = False
     skip: StrictBool = False
     vcs_ref: OptStr
+    migrations: Sequence[Migrations] = ()
 
     def __init__(self, **data: Any):
         super().__init__(**data)

copier/main.py

@@ -13,7 +13,15 @@ from plumbum.cmd import git
 from . import vcs
 from .config import make_config
 from .config.objects import ConfigData, UserMessageError
-from .tools import Renderer, Style, copy_file, get_name_filters, make_folder, printf
+from .tools import (
+    Renderer,
+    Style,
+    copy_file,
+    get_migration_tasks,
+    get_name_filters,
+    make_folder,
+    printf,
+)
 from .types import (
     AnyByStrDict,
     CheckPathFunc,
@@ -43,7 +51,7 @@ def copy(
     skip: OptBool = False,
     quiet: OptBool = False,
     cleanup_on_error: OptBool = True,
-    vcs_ref: str = "HEAD",
+    vcs_ref: OptStr = None,
     only_diff: OptBool = True,
 ) -> None:
     """
@@ -167,10 +175,9 @@ def copy_local(conf: ConfigData) -> None:
     if not conf.quiet:
         print("")  # padding space
 
-    if conf.tasks:
+    run_tasks(conf, render, conf.tasks)
-        run_tasks(conf, render)
+    if not conf.quiet:
-        if not conf.quiet:
+        print("")  # padding space
-            print("")  # padding space
 
 
 def update_diff(conf: ConfigData):
@@ -207,11 +214,16 @@ def update_diff(conf: ConfigData):
             git("remote", "add", "real_dst", conf.dst_path)
             git("fetch", "real_dst", "HEAD")
             diff = git("diff", "--unified=0", "HEAD...FETCH_HEAD")
+    # Run pre-migration tasks
+    renderer = Renderer(conf)
+    run_tasks(conf, renderer, get_migration_tasks(conf, "before"))
     # Do a normal update in final destination
     copy_local(conf)
     # Try to apply cached diff into final destination
     with local.cwd(conf.dst_path):
         (git["apply", "--reject"] << diff)(retcode=None)
+    # Run post-migration tasks
+    run_tasks(conf, renderer, get_migration_tasks(conf, "after"))
 
 
 def get_source_paths(
@@ -310,9 +322,9 @@ def overwrite_file(conf: ConfigData, dst_path: Path, rel_path: Path) -> bool:
     return bool(ask(f" Overwrite {dst_path}?", default=True))
 
 
-def run_tasks(conf: ConfigData, render: Renderer) -> None:
+def run_tasks(conf: ConfigData, render: Renderer, tasks: StrSeq) -> None:
-    for i, task in enumerate(conf.tasks):
+    for i, task in enumerate(tasks):
         task = render.string(task)
         # TODO: should we respect the `quiet` flag here as well?
-        printf(f" > Running task {i + 1} of {len(conf.tasks)}", task, style=Style.OK)
+        printf(f" > Running task {i + 1} of {len(tasks)}", task, style=Style.OK)
         subprocess.run(task, shell=True, check=True, cwd=conf.dst_path)

copier/tools.py

@@ -10,11 +10,20 @@ from typing import Any, Optional, Sequence, Tuple, Union
 import colorama
 from jinja2 import FileSystemLoader
 from jinja2.sandbox import SandboxedEnvironment
+from packaging import version
 from pydantic import StrictBool
 from ruamel.yaml import round_trip_dump
 
 from .config.objects import ConfigData, EnvOps
-from .types import AnyByStrDict, CheckPathFunc, IntSeq, JSONSerializable, StrOrPath, T
+from .types import (
+    AnyByStrDict,
+    CheckPathFunc,
+    IntSeq,
+    JSONSerializable,
+    StrOrPath,
+    StrSeq,
+    T,
+)
 
 __all__ = ("Style", "printf")
 
@@ -173,3 +182,19 @@ def get_name_filters(
         return must_be_filtered(path) and not must_be_included(path)
 
     return must_filter, must_skip
+
+
+def get_migration_tasks(conf: ConfigData, stage: str) -> StrSeq:
+    """Get migration objects that match current version spec.
+
+    Versions are compared using PEP 440.
+    """
+    result: StrSeq = []
+    if not conf.old_commit or not conf.commit:
+        return result
+    vfrom = version.parse(conf.old_commit)
+    vto = version.parse(conf.commit)
+    for migration in conf.migrations:
+        if vto >= version.parse(migration.version) > vfrom:
+            result += migration.dict().get(stage, [])
+    return result

copier/vcs.py

@@ -3,10 +3,11 @@ import shutil
 import tempfile
 from pathlib import Path
 
-from plumbum import TF, local
+from packaging import version
+from plumbum import TF, colors, local
 from plumbum.cmd import git
 
-from .types import OptStr
+from .types import OptStr, StrOrPath
 
 __all__ = ("get_repo", "clone")
 
@@ -52,6 +53,21 @@ def get_repo(url: str) -> OptStr:
     return url
 
 
+def checkout_latest_tag(local_repo: StrOrPath) -> str:
+    """Checkout latest git tag and check it out, sorted by PEP 440."""
+    with local.cwd(local_repo):
+        all_tags = git("tag").split()
+        sorted_tags = sorted(all_tags, key=version.parse, reverse=True)
+        try:
+            latest_tag = str(sorted_tags[0])
+        except IndexError:
+            print(colors.warn | "No git tags found in template; using HEAD as ref")
+            latest_tag = "HEAD"
+        git("checkout", "--force", latest_tag)
+        git("submodule", "update", "--checkout", "--init", "--recursive", "--force")
+        return latest_tag
+
+
 def clone(url: str, ref: str = "HEAD") -> str:
     location = tempfile.mkdtemp()
     shutil.rmtree(location)  # Path must not exist

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 combine_as_imports = true
 force_grid_wrap = 0
 include_trailing_comma = true
-known_third_party = ["colorama", "jinja2", "pkg_resources", "plumbum", "pydantic", "pytest", "ruamel", "setuptools", "six"]
+known_third_party = ["colorama", "jinja2", "packaging", "pkg_resources", "plumbum", "pydantic", "pytest", "ruamel", "setuptools", "six"]
 line_length = 88
 multi_line_output = 3
 use_parentheses = true

setup.cfg

@@ -28,6 +28,7 @@ install_requires =
     ruamel.yaml ~= 0.15
     plumbum ~= 1.6
     pydantic >= 1.0b1
+    packaging ~= 20.1
 
 [options.packages.find]
 exclude =

tests/test_config.py

@@ -175,6 +175,7 @@ def test_config_data_good_data(dst):
         "quiet": False,
         "skip": False,
         "vcs_ref": None,
+        "migrations": (),
     }
     conf = ConfigData(**expected)
     expected["data"]["_folder_name"] = dst.name
@@ -214,5 +215,5 @@ def test_make_config_good_data(dst):
     ],
 )
 def test_make_config_precedence(dst, test_input, expected):
-    conf = make_config(dst_path=dst, **test_input)
+    conf = make_config(dst_path=dst, vcs_ref="HEAD", **test_input)
     assert is_subdict(expected, conf.dict())

tests/test_copy.py

@@ -50,7 +50,7 @@ def test_copy(dst):
 
 
 def test_copy_repo(dst):
-    copier.copy("gh:jpscaletti/siht.git", dst, quiet=True)
+    copier.copy("gh:jpscaletti/siht.git", dst, vcs_ref="HEAD", quiet=True)
     assert (dst / "setup.py").exists()
 
 

tests/test_updatediff.py

@@ -8,8 +8,6 @@ from copier.cli import CopierApp
 
 from .helpers import PROJECT_TEMPLATE
 
-COMMIT_1 = "49deace1b66f3a88a6305cc380d7596cc8170dc9"
-COMMIT_2 = "c2ac5c45404cbd9b031acebcf398f19f56ce49dc"
 REPO_BUNDLE_PATH = Path(f"{PROJECT_TEMPLATE}_updatediff_repo.bundle").absolute()
 
 
@@ -18,15 +16,15 @@ def test_updatediff(dst: Path):
     readme = target / "README.txt"
     answers = target / ".copier-answers.yml"
     commit = git["commit", "--all", "--author", "Copier Test <test@copier>"]
-    # Run copier 1st time, with specific commit
+    # Run copier 1st time, with specific tag
     CopierApp.invoke(
-        "copy", str(REPO_BUNDLE_PATH), str(target), force=True, vcs_ref=COMMIT_1
+        "copy", str(REPO_BUNDLE_PATH), str(target), force=True, vcs_ref="v0.0.1"
     )
     # Check it's copied OK
     assert answers.read_text() == dedent(
         f"""
             # Changes here will be overwritten by Copier
-            _commit: {COMMIT_1}
+            _commit: v0.0.1
             _src_path: {REPO_BUNDLE_PATH}
             author_name: Guybrush
             project_name: to become a pirate
@@ -46,28 +44,55 @@ def test_updatediff(dst: Path):
         git("init")
         git("add", ".")
         commit("-m", "hello world")
-    # Emulate the user modifying the README by hand
+        # Emulate the user modifying the README by hand
-    with open(readme, "w") as readme_fd:
+        with open(readme, "w") as readme_fd:
-        readme_fd.write(
+            readme_fd.write(
-            dedent(
+                dedent(
-                """
+                    """
-                Let me introduce myself.
+                    Let me introduce myself.
 
-                My name is Guybrush, and my project is to become a pirate.
+                    My name is Guybrush, and my project is to become a pirate.
 
-                Thanks for your grog.
+                    Thanks for your grog.
-                """
+                    """
+                )
             )
-        )
-    with local.cwd(target):
         commit("-m", "I prefer grog")
-        # Update target to latest commit
+        # Update target to latest tag and check it's updated in answers file
         CopierApp.invoke(force=True)
+        assert answers.read_text() == dedent(
+            f"""
+                # Changes here will be overwritten by Copier
+                _commit: v0.0.2
+                _src_path: {REPO_BUNDLE_PATH}
+                author_name: Guybrush
+                project_name: to become a pirate
+            """
+        )
+        # Check migrations were executed properly
+        assert not (target / "before-v0.0.1").is_file()
+        assert not (target / "after-v0.0.1").is_file()
+        assert (target / "before-v0.0.2").is_file()
+        assert (target / "after-v0.0.2").is_file()
+        (target / "before-v0.0.2").unlink()
+        (target / "after-v0.0.2").unlink()
+        assert not (target / "before-v1.0.0").is_file()
+        assert not (target / "after-v1.0.0").is_file()
+        commit("-m", "Update template to v0.0.2")
+        # Update target to latest commit, which is still untagged
+        CopierApp.invoke(force=True, vcs_ref="HEAD")
+        # Check no new migrations were executed
+        assert not (target / "before-v0.0.1").is_file()
+        assert not (target / "after-v0.0.1").is_file()
+        assert not (target / "before-v0.0.2").is_file()
+        assert not (target / "after-v0.0.2").is_file()
+        assert not (target / "before-v1.0.0").is_file()
+        assert not (target / "after-v1.0.0").is_file()
     # Check it's updated OK
     assert answers.read_text() == dedent(
         f"""
             # Changes here will be overwritten by Copier
-            _commit: {COMMIT_2}
+            _commit: v0.0.2-1-g81c335d
             _src_path: {REPO_BUNDLE_PATH}
             author_name: Guybrush
             project_name: to become a pirate