Automatic Buildout Updates
==========================

When a buildout is run, one of the first steps performed is to check
for updates to either zc.buildout or setuptools.  To demonstrate this,
we've created some "new releases" of buildout and setuptools in a
new_releases folder:

    >>> ls(new_releases)
    d  setuptools
    -  setuptools-1.99.99-py2.4.egg
    d  zc.buildout
    -  zc.buildout-1.100.0b1-pyN.N.egg
    -  zc.buildout-1.99.99-py2.4.egg
    -  zc.buildout-2.0.0-pyN.N.egg

Let's update the sample buildout.cfg to look in this area:

    >>> write(sample_buildout, 'buildout.cfg',
    ... """
    ... [buildout]
    ... find-links = %(new_releases)s
    ... index = %(new_releases)s
    ... parts = show-versions
    ... develop = showversions
    ...
    ... [show-versions]
    ... recipe = showversions
    ... """ % dict(new_releases=new_releases))

We'll also include a recipe that echos the versions of setuptools and
zc.buildout used:

    >>> mkdir(sample_buildout, 'showversions')

    >>> write(sample_buildout, 'showversions', 'showversions.py',
    ... """
    ... import pkg_resources
    ...
    ... class Recipe:
    ...
    ...     def __init__(self, buildout, name, options):
    ...         pass
    ...
    ...     def install(self):
    ...         for project in 'zc.buildout', 'setuptools':
    ...             req = pkg_resources.Requirement.parse(project)
    ...             print project, pkg_resources.working_set.find(req).version
    ...         return ()
    ...     update = install
    ... """)


    >>> write(sample_buildout, 'showversions', 'setup.py',
    ... """
    ... from setuptools import setup
    ...
    ... setup(
    ...     name = "showversions",
    ...     entry_points = {'zc.buildout': ['default = showversions:Recipe']},
    ...     )
    ... """)


Now if we run the buildout, the buildout will upgrade itself to the
new versions found in new releases:

    >>> print system(buildout),
    Getting distribution for 'zc.buildout>=1.99, <2dev'.
    Got zc.buildout 1.99.99.
    Getting distribution for 'setuptools'.
    Got setuptools 1.99.99.
    Upgraded:
      zc.buildout version 1.99.99,
      setuptools version 1.99.99;
    restarting.
    Generated script '/sample-buildout/bin/buildout'.
    Develop: '/sample-buildout/showversions'
    Installing show-versions.
    zc.buildout 1.99.99
    setuptools 1.99.99

Notice that, even though we have a newer beta version of zc.buildout
available, the final "1.99.99" was selected.  If you want to get non-final
versions, specify a specific version in your buildout's versions
section, you typically want to use the --accept-buildout-test-releases
option to the bootstrap script, which internally uses the
``accept-buildout-test-releases = true`` discussed below.

Also, even thought there's a later final version, buildout won't
upgrade itself past version 1.

Our buildout script's site.py has been updated to use the new eggs:

    >>> cat(sample_buildout, 'parts', 'buildout', 'site.py')
    ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
    "...
    def addsitepackages(known_paths):
        """Add site packages, as determined by zc.buildout.
    <BLANKLINE>
        See original_addsitepackages, below, for the original version."""
        setuptools_path = '/sample-buildout/eggs/setuptools-1.99.99-pyN.N.egg'
        sys.path.append(setuptools_path)
        known_paths.add(os.path.normcase(setuptools_path))
        import pkg_resources
        buildout_paths = [
            '/sample-buildout/eggs/zc.buildout-1.99.99-pyN.N.egg',
            '/sample-buildout/eggs/setuptools-1.99.99-pyN.N.egg'
            ]
        for path in buildout_paths:
            sitedir, sitedircase = makepath(path)
            if not sitedircase in known_paths and os.path.exists(sitedir):
                sys.path.append(sitedir)
                known_paths.add(sitedircase)
                pkg_resources.working_set.add_entry(sitedir)
        sys.__egginsert = len(buildout_paths) # Support setuptools.
        original_paths = [
            ...
            ]
        for path in original_paths:
            if path == setuptools_path or path not in known_paths:
                addsitedir(path, known_paths)
        return known_paths
    ...

Now, let's recreate the sample buildout. If we specify constraints on
the versions of zc.buildout and setuptools (or distribute) to use,
running the buildout will install earlier versions of these packages:

    >>> write(sample_buildout, 'buildout.cfg',
    ... """
    ... [buildout]
    ... find-links = %(new_releases)s
    ... index = %(new_releases)s
    ... parts = show-versions
    ... develop = showversions
    ... zc.buildout-version = < 1.99
    ... setuptools-version = < 1.99
    ... distribute-version = < 1.99
    ...
    ... [show-versions]
    ... recipe = showversions
    ... """ % dict(new_releases=new_releases))

Now we can see that we actually "upgrade" to an earlier version.

    >>> print system(buildout),
    Upgraded:
      zc.buildout version 1.0.0,
      setuptools version 0.6;
    restarting.
    Develop: '/sample-buildout/showversions'
    Updating show-versions.
    zc.buildout 1.0.0
    setuptools 0.6

There are a number of cases, described below, in which the updates
don't happen.

We won't upgrade in offline mode:

    >>> write(sample_buildout, 'buildout.cfg',
    ... """
    ... [buildout]
    ... find-links = %(new_releases)s
    ... index = %(new_releases)s
    ... parts = show-versions
    ... develop = showversions
    ...
    ... [show-versions]
    ... recipe = showversions
    ... """ % dict(new_releases=new_releases))

    >>> print system(buildout+' -o'),
    Develop: '/sample-buildout/showversions'
    Updating show-versions.
    zc.buildout 1.0.0
    setuptools 0.6

Or in non-newest mode:

    >>> print system(buildout+' -N'),
    Develop: '/sample-buildout/showversions'
    Updating show-versions.
    zc.buildout 1.0.0
    setuptools 0.6

We also won't upgrade if the buildout script being run isn't in the
buildout's bin directory.  To see this we'll create a new buildout
directory:

    >>> sample_buildout2 = tmpdir('sample_buildout2')
    >>> write(sample_buildout2, 'buildout.cfg',
    ... """
    ... [buildout]
    ... find-links = %(new_releases)s
    ... index = %(new_releases)s
    ... parts =
    ... """ % dict(new_releases=new_releases))

    >>> cd(sample_buildout2)
    >>> print system(buildout),
    Creating directory '/sample_buildout2/bin'.
    Creating directory '/sample_buildout2/parts'.
    Creating directory '/sample_buildout2/eggs'.
    Creating directory '/sample_buildout2/develop-eggs'.
    Getting distribution for 'zc.buildout>=1.99, <2dev'.
    Got zc.buildout 1.99.99.
    Getting distribution for 'setuptools'.
    Got setuptools 1.99.99.
    Not upgrading because not running a local buildout command.

    >>> ls('bin')

As mentioned above, the ``accept-buildout-test-releases = true`` means that
newer non-final versions of these dependencies are preferred.  Typically
users are not expected to actually manipulate this value.  Instead, the
bootstrap script creates a buildout buildout script that passes in the
value as a command line override. This then results in the buildout
script being rewritten to remember the decision.

We'll mimic this by passing the argument actually in the command line.

    >>> cd(sample_buildout)
    >>> write(sample_buildout, 'buildout.cfg',
    ... """
    ... [buildout]
    ... find-links = %(new_releases)s
    ... index = %(new_releases)s
    ... parts = show-versions
    ... develop = showversions
    ...
    ... [show-versions]
    ... recipe = showversions
    ... """ % dict(new_releases=new_releases))

    >>> print system(buildout +
    ...              ' buildout:accept-buildout-test-releases=true'),
    ... # doctest: +NORMALIZE_WHITESPACE
    Getting distribution for 'zc.buildout>=1.99, <2dev'.
    Got zc.buildout 1.100.0b1.
    Upgraded:
      zc.buildout version 1.100.0b1,
      setuptools version 1.99.99;
    restarting.
    Generated script '/sample-buildout/bin/buildout'.
    NOTE: Accepting early releases of build system packages.  Rerun bootstrap
          without --accept-buildout-test-releases (-t) to return to default
          behavior.
    Develop: '/sample-buildout/showversions'
    Updating show-versions.
    zc.buildout 1.100.0b1
    setuptools 1.99.99

The buildout script shows the change.

    >>> buildout_script = join(sample_buildout, 'bin', 'buildout')
    >>> import sys
    >>> if sys.platform.startswith('win'):
    ...     buildout_script += '-script.py'
    >>> print open(buildout_script).read() # doctest: +ELLIPSIS
    #...
    sys.argv.insert(1, 'buildout:accept-buildout-test-releases=true')
    print ('NOTE: Accepting early releases of build system packages.  Rerun '
           'bootstrap without --accept-buildout-test-releases (-t) to return to '
           'default behavior.')
    ...

If the update process for buildout or setuptools fails the error should be
caught (displaying a warning) and the rest of the buildout update process
should continue.

    >>> version = sys.version_info[0:2]
    >>> egg = new_releases + '/zc.buildout-1.99.99-py%s.%s.egg' % version
    >>> copy_egg = new_releases + '/zc.buildout-1.1000-py%s.%s.egg' % version
    >>> import shutil
    >>> shutil.copy(egg, copy_egg)

Create a broken egg

    >>> mkdir(sample_buildout, 'broken')
    >>> write(sample_buildout, 'broken', 'setup.py', "import broken_egg\n")
    >>> write(sample_buildout, 'buildout.cfg',
    ... """
    ... [buildout]
    ... find-links = %(new_releases)s
    ... index = %(new_releases)s
    ... parts = show-versions
    ... develop =
    ...   broken
    ...
    ... [broken]
    ... recipe = zc.recipe.egg
    ... eggs = broken
    ... """ % dict(new_releases=new_releases))
    >>> import subprocess
    >>> subprocess.call([buildout])
    1
