Useful muddle classes, methods and functions

This is the start of a section describing just those parts of the muddle API that are generally likely to be useful in build descriptions. For the moment, much of it is just duplicates of the docstring descriptions already found in the next section, The muddled package.

An example build description

#! /usr/bin/env python

"""Muddle build description for a "plain" OMAP build on the Beagleboard.

At least initially, we're trying to build a Linux+busybox system.

The role we're providing is 'omap'.
"""

import os

import muddled.deployments.filedep as filedep
import muddled.pkgs.aptget as aptget
import muddled.checkouts.simple
import muddled.depend
from muddled.depend import label_from_string
import muddled.pkgs.make as make

import wget
import repo

def describe_to(builder):

    role = 'omap'
    roles = ['omap']

    builder.add_default_role(role)

    # filedep.deploy(builder, target_dir, name, roles)
    #
    # Register a file deployment.
    #
    # The deployment will take the roles specified in the role list, and build
    # them into a deployment at deploy/[name].
    #
    # The deployment should eventually be located at target_dir.

    filedep.deploy(builder,
                   '/',
                   'omap',
                   roles)

    # There's a variety of things we need on (this, the host) system
    # in order to build - I hope I've got this right (difficult to tell
    # as I already have some stuff installed on *my* development system)
    aptget.simple(builder, 'development_packages', role,
            ['zlib1g-dev', 'uboot-mkimage'])

    # According to http://omappedia.org/wiki/LinuxOMAP_Kernel_Project, we get
    # our OMAP kernel from:
    muddled.checkouts.simple.absolute(builder, 'omap_kernel',
            'git+git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap-2.6.git')

    # Once we've got one worked out, we'll also want to retrieve a default
    # kernel configuration (we don't, eventually, want to make the developer
    # have to work that out every time!)

    # We'll aim to make that with an out-of-tree makefile
    # We could make a tailored subclass of muddled.pkgs.linux_kernel, and use
    # that to build our kernel. I may still do that once I've figured out how
    # it is different (one notable change is we're building uImage instead of
    # zImage). For now, it's probably easier to have a Makefile.muddle
    make.medium(builder,
                name = "omap_kernel",    # package name
                roles = roles,
                checkout = "builders",
                deps = [],
                makefileName = os.path.join("omap_kernel","Makefile.muddle"))

    muddled.pkg.package_depends_on_checkout(builder.ruleset,
                                    "omap_kernel",  # this package
                                    role,           # in this role
                                    "omap_kernel")  # depends on this checkout

    # On top of that, we want to build busybox
    #
    # According to the busybox website, we can retrieve sources via git:
    #
    # To grab a copy of the BusyBox repository using anonymous git access::
    #
    #   git clone git://busybox.net/busybox.git
    #
    # Once you have the repository, stable branches can be checked out by
    # doing::
    #
    #   git checkout remotes/origin/1_NN_stable
    #
    # Once you've checked out a copy of the source tree, you can update your
    # source tree at any time so it is in sync with the latest and greatest by
    # entering your BusyBox directory and running the command::
    #
    #   git pull

    # So, for the moment, at least, let's go with the latest from source
    # control (when we're finalising this, we're maybe better identifying
    # a particular release to stick to)
    muddled.checkouts.simple.absolute(builder, 'busybox',
            'git+git://busybox.net/busybox.git')

    # We'll aim to make that with an out-of-tree makefile
    #
    # 'deps' is a list of package names, which our make depends on.
    #
    # Specifically, for each <name> in 'deps', and for each <role> in 'roles',
    # we will depend on "package:<name>{<role>}/postinstalled"
    make.medium(builder,
                name = "busybox",    # package name
                roles = roles,
                checkout = "builders",
                deps = [ 'omap_kernel' ],
                makefileName = os.path.join("busybox","Makefile.muddle"))

    # And we also depend on having actually checked out busybox
    muddled.pkg.package_depends_on_checkout(builder.ruleset,
                                    "busybox",      # this package
                                    role,           # in this role
                                    "busybox")      # depends on this checkout


    # Can we use the same bootloader and such that we already had for the
    # android build?

    # The bootloader and related items, which go into the FAT32 partition on
    # the flash card, are retrieved from the net (eventually, I hope we'll be
    # building u-boot, but for now the binary should do)
    muddled.checkouts.simple.absolute(builder, 'MLO',
            'wget+http://free-electrons.com/pub/demos/beagleboard/android/MLO')
    muddled.checkouts.simple.absolute(builder, 'u-boot',
            'wget+http://free-electrons.com/pub/demos/beagleboard/android/u-boot.bin')

    # We need some way of getting them installed - let's foreshadow the day when
    # we actually want to build u-boot ourselves, and pretend
    make.medium(builder,
                name = "u-boot",    # package name
                roles = roles,
                checkout = "builders",
                deps = [],
                makefileName = os.path.join("u-boot","Makefile.muddle"))
    muddled.pkg.package_depends_on_checkout(builder.ruleset,
                                    "u-boot",       # this package
                                    role,           # in this role
                                    "u-boot")       # depends on this checkout
    # Oh, and this one as well...
    rule = muddled.depend.depend_one(None,
                              label_from_string('package:u-boot/installed'),
                              label_from_string('checkout:MLO/checked_out'))
    builder.ruleset.add(rule)


    # And, of course, we need (the rest of) our Linux filesystem
    muddled.checkouts.simple.absolute(builder, 'rootfs',
            'bzr+ssh://bzr@palmera.c.kynesim.co.uk//opt/kynesim/projects/052/rootfs')
    make.simple(builder, 'rootfs', role, 'rootfs', config=False,
            makefileName='Makefile.muddle')

    # But we depend on busybox to have installed the various binaries first
    rule = muddled.depend.depend_one(None,
                              label_from_string('package:rootfs/installed'),
                              label_from_string('package:busybox/installed'))
    builder.ruleset.add(rule)

    # Deploy all our roles
    builder.by_default_deploy_list(roles)

muddled.mechanics.Builder

The describe_to() function in a build description takes a Builder instance as its argument. Thus the methods on the Builder class are all simply available. For instance:

builder.by_default_deploy_list(roles)

class muddled.mechanics.Builder(root_path, muddle_binary, domain_params=None, default_domain=None)

Bases: object

A builder does stuff following rules derived from a build description.

Don’t construct a Builder directly, always use the ‘load_builder()’ function, or ‘minimal_build_tree()’ if that is more appropriate.

• self.db - The metadata database for this project.
• self.ruleset - The rules describing this build
• self.env - A dictionary of label to environment
• self.default_roles - The roles to build when you don’t specify any. These will also be used for “guessing” a role for a package when one is not specified. ‘_default_roles’ is calculated from this.
• self.default_deployment_labels - The deployments to deploy when you don’t specify any specific roles. This is the meaning of ‘_default_deployments’.

There are then a series of values used in managing subdomains:

• self.banned_roles - An array of pairs of the form (role, domain) which aren’t allowed to share libraries.
• self.domain_params - Maps domain names to dictionaries storing parameters that other domains can retrieve. This is used to communicate values from a build to its subdomains.
• self.unifications - This is a list of tuples of the form (source-label, target-label), where one “replaces” the other in the build tree.

And:

• self.what_to_release - a set of entities to build for a release build. It may contain labels and also “special” names, such as ‘_default_deployments’ or even ‘_all’. You may not include ‘_release’ (did you need to ask?). “_just_pulled” is not allowed either.

Construct a fresh Builder with a .muddle directory at the given ‘root_path’.

‘muddle_binary’ is the full path to our muddle “binary” - the program that is muddle. This is needed when running muddle Makefiles that invoke $(MUDDLE).

‘domain_params’ is the set of domain parameters in effect when this builder is loaded. It’s used to communicate values down to sub-domains.

Note that you MUST NOT set ‘domain_params’ Null unless you are the top-level domain - it MUST come from the enclosing domain’s builder or modifications made by the subdomain’s buidler will be lost, and as this is the only way to communicate values to a parent domain, this would be bad. Ugh.

‘default_domain’ is the default domain value to add to anything in local_pkgs , etc - it’s used to make sure that if you’re cd’d into a domain subdirectory, we build the right labels.

add_default_deployment_label(label)

Set the label that’s built when you call muddle from the root directory

add_default_role(role)

Add role to the list of roles built when you don’t ask for something different.

Returns False if we didn’t actually add the role (it was already there), True if we did.

add_default_roles(roles)

Add the given roles to the list of default roles for this build.

add_to_release_build(thing)

Add a thing to the set of entities to build for a release build.

‘thing’ must be:

• a Label
• one of the “special” names, “_all”, “_default_deployments”, “_default_roles”.
• a sequence of either/both

It may not be “_release” (!) or “_just_pulled”.

Special names are expanded after all build descriptions have been read.

The special name “_release” corresponds to this set.

all_checkout_labels(tag=None)

Return a set of the labels of all the checkouts in our rule set.

Note that if ‘tag’ is None then all the labels will be of the form:

checkout:<co_name>/*

otherwise ‘tag’ will be used as the checkout label tag:

checkout:<co_name>/<tag>

all_checkout_rules()

Returns a set of the labels of all the checkouts in our rule set.

Specifically, all the rules for labels of the form:

checkout:*{*}/checked_out
checkout:(*)*{*}/checked_out

Returns a set of labels, thus allowing one to know the domain of each checkout as well as its name.

all_checkouts()

Return a set of the names of all the checkouts in our rule set.

Returns a set of strings.

This is not domain aware. Consider using all_checkout_labels(), which is.

all_deployment_labels(required_tag)

Return a set of all the deployment labels in our ruleset.

If ‘required_tag’ is given, then the labels returned will all have that tag (this, of course, may result in a smaller set of labels).

all_deployments()

Return a set of the names of all the deployments in our rule set.

Returns a set of strings.

all_domains()

Return a set of the names of all the domains in our rule set.

Returns a set of strings. The ‘None’ domain (the unnamed, top-level domain) is returned as the empty string, “”.

all_package_labels()

Return a set of the labels of all the packages in our rule set.

all_packages()

Return a set of the names of all the packages in our rule set.

Returns a set of strings.

Note that if ‘*’ is one of the package “names” in the ruleset, then it will be included in the names returned.

all_packages_with_roles()

Return a set of the names of all the packages/roles in our rule set.

Returns a set of strings.

Note that if ‘*’ is one of the package “names” in the ruleset, then it will be included in the names returned.

However, any labels with role ‘*’ will be ignored.

all_roles()

Return a set of the names of all the roles in our rule set.

Returns a set of strings.

apply_unifications(source)

build_co_and_path()

Return a pair (build_co, build_path).

build_desc_repo

The Repository for our build description.

This used to be a simple value, but is now a shim around looking it up in builder.db.checkout_data - so that we only have the information stored in one place.

Returns None if there is no Repository registered yet

build_label(label, silent=False)

The fundamental operation of a builder - build this label.

build_label_with_options(label, useDepends=True, useTags=True, silent=False)

The fundamental operation of a builder - build this label.

• useDepends - Use dependencies?

build_name

The build name is meant to be a short description of the purpose of a build. It might thus be something like “ProjectBlue_intel_STB” or “XWing-minimal”.

The name may only contain alphanumerics, underlines and hyphens - this is to facilitate its use in version stamp filenames. Also, it is a superset of the allowed characters in a Python module name, which means that the build description filename (excluding its ”.py”) will be a legal build name (so we can use that as a default).

by_default_deploy(deployment)

Add a deployment label to the deployments to build by default.

by_default_deploy_list(deployments)

Now we’ve got a list of default labels, we can just add them ..

checkout_label_exists(label)

Return True if this checkout label is in any rules (i.e., is used).

Note that this method does not understand wildcards, so the match must be exact.

checkout_path(label)

Return the path in which the given checkout resides.

This is a simple wrapper around builder.db.get_checkout_path(), provided for use in scripts and build descriptions, since it “matches” builder.package_obj_path, builder.deploy_path, and so on.

checkouts_for_package(pkg_label)

Return a set of the checkouts that the given package depends upon

This only looks at direct dependencies (so if a package depends on something that in turn depends on a checkout that it does not directly depend on, then that indirect checkout will not be returned).

It does, however, expand wildcards.

deploy_path(label)

Where should deployment ‘label’ deploy to?

diagnose_unused_labels(labels, arg, required_type=None, required_tag='*')

Concoct a useful report on why none of ‘labels’ is used.

We rely on ‘labels’ having been generated by our label_from_fragment() method, which means that all the labels will have the same type

We assume quite a lot of knowledge about how that method works.

effective_environment_for(label)

Return an environment which embodies the settings that should be used for the given label. It’s the in-order merge of the output of list_environments_for().

expand_release()

Expand the command line argument “_release”

expand_underscore_arg(word, type_for_all=None)

Given a command line argument (‘word’) that starts with an underscore, try to expand it to a list of labels.

If the argument is _all, then if ‘type_for_all’ is given, expand it to all labels of that type, and otherwise reject it.

Raises a GiveUp exception if the argument is not recognised.

expand_wildcards(label, default_to_obvious_tag=True)

Given a label which may contain wildcards, return a set of labels that match.

As per the normal definition of labels, the <type>, <name>, <role> and <tag> parts of the label may be wildcarded.

If default_to_obvious_tag is true, then if label has a tag of ‘*’, it will be replaced by the “obvious” (final) tag for this label type, before any searching (so for a checkout: label, /checked_out would be used).

find_local_package_labels(dir, tag)

This is slightly horrible because if you’re in a source checkout (as you normally will be), there could be several packages.

Returns a list of the package labels involved. Uses the given tag for the labels.

find_location_in_tree(dir)

Find the directory type and name of subdirectory in a repository. This is used by the find_local_package_labels method to work out which packages to rebuild

• dir - The directory to analyse

If nothing sensible can be determined, we return None. Otherwise we return a tuple of the form:

(DirType, label, domain_name)

where:

• ‘DirType’ is a utils.DirType value,
• ‘label’ is None or a label describing our location,
• ‘domain_name’ None or the subdomain we are in and

If ‘label’ and ‘domain_name’ are both given, they will name the same domain.

follow_build_desc_branch

follows_build_desc_branch

get_all_checkout_labels_below(dir)

Get the labels of all the checkouts in or below directory ‘dir’

NOTE that this will not work if you are in a subdirectory of a checkout. It’s not meant to. Consider using find_location_in_tree() to determine that, before calling this method.

get_build_desc_branch(verbose=False)

Return the current branch of the top-level build description.

(Returns None if the build description is not on a branch, or if its VCS does not support this operation.)

get_default_domain()

get_dependent_package_dirs(label)

Find all the dependent packages for label and return a set of the object directories for each. Mainly used as a helper function by set_default_variables().

get_distribution()

Retrieve the current distribution name and target directory.

Raises GiveUp if there is no current distribution set.

get_domain_parameter(domain, name)

get_domain_parameters(domain)

get_environment_for(label)

Return the environment store for the given label, inventing one if necessary.

get_labels_in_default_roles()

Return a list of the package labels in the default roles.

get_parameter(name)

Returns the given domain parameter, or None if it wasn’t defined.

include_domain(domain_builder, domain_name)

Import the builder domain_builder into the current builder, giving it domain_name.

We first import the db, then we rename None to domain_name in banned_roles

instruct(pkg, role, instruction_file, domain=None)

Register the existence or non-existence of an instruction file. If instruction_file is None, we unregister the instruction file.

• instruction_file - A db.InstructionFile object to save.

is_release_build()

Are we a release build (i.e., a build tree created by “muddle release”)?

We look to see if there is a file called .muddle/Release

kill_label(label, useTags=True, useMatch=True)

Kill everything that matches the given label and all its consequents.

label_from_fragment(fragment, default_type)

A variant of Label.from_fragment that understands types and wildcards

In particular, it knows that:

1. packages have roles, but checkouts and deployments do not.
2. wildcards expand to their appropriate values

Returns a list of labels. This method does not check that all of the labels returned actually exist as targets in the dependency tree.

labels_for_role(kind, role, tag, domain=None)

Find all the target labels with the specified kind, role and tag and return them in a set.

If ‘domain’ is specified, also require the domain to match.

list_environments_for(label)

Return a list of environments that contribute to the environment for the given label.

Returns a list of triples (match level, label, environment), in order.

load_instructions(label)

Load the instructions which apply to the given label (usually a wildcard on a role, from a deployment) and return a list of triples (label, filename, instructionfile).

map_unifications(source_list)

mark_domain(domain_name)

Write a file that marks this directory as a domain so we don’t mistake it for the root.

note_unification(source, target)

package_install_path(label)

Where should pkg install itself, by default?

package_obj_path(label)

Where should the package with this label build its object files?

packages_for_deployment(dep_label)

Return a set of the packages that the given deployment depends upon

This only looks at direct dependencies (so if a deployment depends on something that in turn depends on a package that it does not directly depend on, then that indirect package will not be returned).

It does, however, expand wildcards.

packages_using_checkout(co_label)

Return a set of the packages which directly use a checkout (this does not include dependencies)

print_banned_roles()

resource_body(file_name)

Return the body of a resource as a string.

resource_file_name(file_name)

role_combination_acceptable_for_lib(r1, r2, domain1=None, domain2=None)

True only if (r1,r2) does not appear in the list of banned roles.

role_install_path(role, domain=None)

Where should this role find its install to deploy?

roles_do_not_share_libraries(r1, r2, domain1=None, domain2=None)

Assert that roles a and b do not share libraries

Either a or b may be * to mean wildcard

Add (r1,r2) to the list of role pairs that do not share their libraries.

set_default_variables(label, store)

Muddle defines a variety of environment variables which are available whilst a label is being built. The particular variables provided depend on the type of label being built, or the type of build.

Package labels are associated with muddle Makefiles, so any environment variable specific to a package label will be available within a muddle Makefile (i.e., commands such as “muddle build” work on package labels).

System Message: SEVERE/4 (/home/docs/checkouts/readthedocs.org/user_builds/muddle/checkouts/latest/muddled/mechanics.py:docstring of muddled.mechanics.Builder.set_default_variables, line 10)

Unexpected section title.

All labels
----------

MUDDLE

The muddle executable itself. This can be used in muddle Makefiles, for instance:

fred_objdir = $(shell $(MUDDLE) query objdir package:fred{base})

MUDDLE_ROOT

The absolute path to the root of the build tree (where the ‘.muddle’ and ‘src’ directories are).

MUDDLE_LABEL

The label currently being built.

MUDDLE_KIND, MUDDLE_NAME, MUDDLE_ROLE, MUDDLE_TAG, MUDDLE_DOMAIN

Broken-down bits of the label being built. Values will not exist if the label does not contain them (so if ‘label’ is a checkout label, MUDDLE_ROLE will not be set).

MUDDLE_OBJ

Where we should build object files for this label - the obj directory for packages, the src directory for checkouts, and the deploy directory for deployments. See “muddle query objdir”.

System Message: SEVERE/4 (/home/docs/checkouts/readthedocs.org/user_builds/muddle/checkouts/latest/muddled/mechanics.py:docstring of muddled.mechanics.Builder.set_default_variables, line 35)

Unexpected section title.

Package labels
--------------

For package labels, we also set:

MUDDLE_OBJ_OBJ, MUDDLE_OBJ_INCLUDE, MUDDLE_OBJ_LIB, MUDDLE_OBJ_BIN

$(MUDDLE_OBJ)/obj, $(MUDDLE_OBJ)/include, $(MUDDLE_OBJ)/lib, $(MUDDLE_OBJ)/bin, respectively. Note that we do not create these directories - it is up to the muddle Makefile to do so.

MUDDLE_INSTALL

Where we should install package files to.

MUDDLE_INSTRUCT

A shortcut to the ‘muddle instruct’ command for this package. Essentially “$(MUDDLE) instruct $(MUDDLE_LABEL)”

MUDDLE_UNINSTRUCT

A shortcut to the ‘muddle uninstruct’ command for this package. Essentially “$(MUDDLE) uninstruct $(MUDDLE_LABEL)”

MUDDLE_PKGCONFIG_DIRS

A path suitable for passing to pkg-config to tell it to look only at packages this label is declared to be dependent on. It will be empty if the label doesn’t have any dependencies.

MUDDLE_PKGCONFIG_DIRS_AS_PATH

The same as MUDDLE_PKGCONFIG_DIRS, for historical reasons.

MUDDLE_INCLUDE_DIRS

A space separated list of include directories, constructed from the packages that this label depends on. Names will have been intelligently escaped for the shell. Only directories that actually exist will be included.

Typically used in a muddle Makefile as:

CFLAGS += $(MUDDLE_INCLUDE_DIRS:%=-I%)

MUDDLE_LIB_DIRS

A space separated list of library directories, constructed from the packages that this label depends on. Names will have been intelligently escaped for the shell. Only directories that actually exist will be included.

Typically used in a muddle Makefile as:

LDFLAGS += $(MUDDLE_LIB_DIRS:%=-L%)

MUDDLE_LD_LIBRARY_PATH

The same values as in MUDDLE_LIB_DIRS, but with items separated by colons. This is useful for passing (as LD_LIBRARY_PATH) to configure scripts that try to look for libraries when linking test programs.

MUDDLE_KERNEL_DIR

If any of the packages that this label depends on has a directory called kerneldir in its obj dir (so, in its own terms, $(MUDDLE_OBJ)/kerneldir), then we set this value to that directory. Otherwise it is not set. If there is more than one candidate, then the last found is used (but the order of search is not defined, so this would be confusing).

If the build tree is building a Linux kernel, it can be useful to build the kernel into a directory of this name.

MUDDLE_KERNEL_SOURCE_DIR

Like MUDDLE_KERNEL_DIR, but it looks for a directory called kernelsource. The same comments apply.

System Message: SEVERE/4 (/home/docs/checkouts/readthedocs.org/user_builds/muddle/checkouts/latest/muddled/mechanics.py:docstring of muddled.mechanics.Builder.set_default_variables, line 105)

Unexpected section title.

Deployment labels
-----------------

For deployment labels we also set:

MUDDLE_DEPLOY_FROM

Where we should deploy from (probably just MUDDLE_INSTALL with the last component removed)

MUDDLE_DEPLOY_TO

Where we should deploy to, if we’re a deployment.

System Message: SEVERE/4 (/home/docs/checkouts/readthedocs.org/user_builds/muddle/checkouts/latest/muddled/mechanics.py:docstring of muddled.mechanics.Builder.set_default_variables, line 116)

Unexpected section title.

Release build values
--------------------

When building in a release tree (typically by use of “muddle release”) extra environment variables are set to allow the build to know useful information about the release. In a non-release build, these will all be set to “(unset)”.

MUDDLE_RELEASE_NAME

The release name.

MUDDLE_RELEASE_VERSION

The release version.

MUDDLE_RELEASE_HASH

The hash of the release stamp file. This acts as a useful unique identifier for a particular release, as it is calculated from the stamp file information describing all the release checkouts.

Two releases with the same name and version, but with different checkout information, will have different release hashes.

set_distribution(name, target_dir)

Set the current distribution name and target directory.

set_domain_parameter(domain, name, value)

set_parameter(name, value)

Set a domain parameter: Danger Will Robinson! This is a

very odd thing to do - domain parameters are typically set by their enclosing domains. Setting your own is an odd idea and liable to get you into trouble. It is, however, the only way of communicating values back from a domain to its parent (and you shouldn’t really be doing that either!)

setup_environment(label, src_env)

Modify src_env to reflect the environments which apply to label, in match order.

target_label_exists(label)

Return True if this label is a target.

If it is not, then we are not going to be able to build it.

Note that this method does not understand wildcards, so the match must be exact.

unify_environments(source, target)

Given a source label and a target label, find all the environments which might apply to source and make them also apply to target.

This is (slightly) easier than one might imagine ..

unify_labels(source, target)

Unify the ‘source’ label with/into the ‘target’ label.

Given a dependency tree containing rules to build both ‘source’ and ‘target’, this edits the tree such that the any occurrences of ‘source’ are replaced by ‘target’, and dependencies are merged as appropriate.

Free variables (i.e. wildcards in the labels) are untouched - if you need to understand that, see depend.py for quite how this works.

Why is it called “unify” rather than “replace”? Mainly because it does more than replacement, as it has to merge the rules/dependencies together. In retrospect, though, some variation on “merge” might have been easier to remember (if also still inaccurate).

uninstruct_all()

Note

The Builder class actually lives in muddled.mechanics, but is available directly as muddled.Builder for convenience.

muddled.depend

Sometimes it is useful to work directly with dependencies. For instance:

rule = muddled.depend.depend_one(None,
         muddled.depend.label_from_string('package:rootfs/installed'),
         muddled.depend.label_from_string('package:busybox/installed'))
builder.ruleset.add(rule)

Dependency sets and dependency management

class muddled.depend.Action

Bases: object

Represents an object you can call to “build” a tag.

build_label(builder, label)

Build the given label. Your dependencies have been satisfied.

• in_deps - Is the set whose dependencies have been satisified.

Returns True on success, False or throw otherwise.

class muddled.depend.Label(type, name, role=None, tag='*', transient=False, system=False, domain=None)

Bases: object

A label denotes an entity in muddle’s dependency hierarchy.

A label is structured as:

<type>:<name>{<role>}/<tag>[<flags>]

or:

<type>:(<domain>)<name>{<role>}/<tag>[<flags>]

The <type>, <name>, <role> and <tag> parts are composed of the characters [A-Za-z0-9-+_], or the wildcard character ‘*’.

The <domain> name is composed of the same plus ‘(‘ and ‘)’.

The domain, role and flags are all optional.

Note

The label strings “type:name/tag” and “type:name{}/tag[]” are identical, although the former is the more usual form.)

The ‘+’ is allowed in label parts to allow for names like “g++”.

Domains are used when relating sub-builds to each other, and are not necessary when relating labels within the same build. It is not allowed to specify the empty domain as “()” - just omit the parentheses.

If necessary “sub-domains” are specified using nested domains – for instance:

(outer)
(outer(inner))
(outer(inner(even.innerer)))

This is intended to be unambiguous rather than pretty.

Note that wildcarding of a domain name currently only supports one level (i.e., the top “(*)”), and not wildcarding of nested domains.

If you do find yourself using multi-level domains, we would strongly suggest reconsidering your overall build design.

The “core” part of a label is the <name>{<role>} or (<domain>)<name>{<role>}. The <type> and <tag> can (typically) be thought of as tracking the progress of the “core” entity through its lifecycle (build sequence, etc.).

Names beginning with an underscore are reserved by muddle, so do not use them for other purposes.

(Why is the ‘domain’ argument at the end of the argument list? Because it was added after the other arguments were already well-established, and some uses of Label use positional arguments.)

Label instances are treated as immutable by the muddle system, although the implementation does not currently enforce this. Please don’t try to abuse this, as Bad Things will happen.

If you’re implementing a new copy-constructor and changing the new instance’s label before returning it, don’t forget to call rehash(). If you don’t, Really Bad Things will happen.

Note

The flags on a label are not immutable, and are regarded as transient annotations.

Note

When a domain is included as a subdomain, all of its labels are “adjusted” to have the new, appropriate domain name. This is clearly a special meaning of the word “immutable”. However, it should only be the muddle system itself doing this.

Because of this (potential change in content of a label), the domain name does not contribute to a label’s hash value. Thus a label that whose domain name is changed will continue to work as the same key in a dictionary (for instance).

Type:What kind of label this is. The standard muddle values are “checkout”, “package” and “deployment”. These values are defined programmatically via muddled.utils.LabelType. Thus the ‘type’ is conventionally used to indicate what general “stage” of the build process a label belongs to. Name:The name of this checkout/package/whatever. This should be a useful mnemonic for the labels purpose. Role:The role for this checkout/package/whatever. A role might delimit the target architecture of the labels it is used in (roles such as “x86”, “arm”, “beagleboard”), or the sort of purpose (“role” in the more traditionale sense, such as “boot”, “firmware”, “packages”), or some other useful delineation of a partition in the general label namespace (thinking of labels as points in an N-dimensional space). Tag:A tag indicating more precisely what stage the label belongs to within each ‘type’. There are different conventional values according to the ‘type’ of the label (for instance, “checked_out”, “built”, “installed”, etc.). These values are defined programmatically via muddled.utils.LabelTag. Transient:If true, changes to this tag will not be persistent in the muddle database. ‘transient’ is used to denote something which will go away when muddle is terminated - e.g. environment variables. System:If true, marks this label as a system label and not to be reported (by ‘muddle depend’) unless asked for. System labels are labels “invented” by muddle itself to satisfy implicit dependencies, or to allow the build system as a whole to work. Domain:The domain is used to specify which build or sub-build this label corresponds to. Nested “tail recursive” parenthesised components may be used to specify sub-domains (but this is not recommended). The domain defaults to the current build.

The role may be None, indicating (for instance) that roles are not relevant to this particular label.

The domain may be None, indicating that the label belongs to the current build. If the domain is given as ‘’ (empty string) then this is equivalent to None, and is stored as such. Do not specify domains unless you need to.

The kind, name, role and tag may be wildcarded, by being set to ‘*’. When evaluating dependencies between labels, for instance, a wildcard indicates “for any value of this part of the label”.

Domains can be wildcarded, and that probably means the obvious (that the label applies across all domains), but this may not yet be implemented. Wildcarding of sub-domains may never be supported.

Note that label flags (including specifically ‘transient’ and ‘system’) are not equality-preserving properties of a label - two labels are not made unequal just because they have different flags.

(In fact, no two labels should ever have different values for transience, for obvious reasons, and the system flag is intended only to limit over-reporting of information.)

For instance:

>>> Label('package', 'busybox')
Label('package', 'busybox', role=None, tag='*')
>>> str(_)
'package:busybox/*'
>>> Label('package', 'busybox', tag='installed')
Label('package', 'busybox', role=None, tag='installed')
>>> str(_)
'package:busybox/installed'
>>> Label('package', 'busybox', role='rootfs', tag='installed')
Label('package', 'busybox', role='rootfs', tag='installed')
>>> str(_)
'package:busybox{rootfs}/installed'
>>> Label('package', 'busybox', 'rootfs', 'installed')
Label('package', 'busybox', role='rootfs', tag='installed')
>>> str(_)
'package:busybox{rootfs}/installed'
>>> Label('package', 'busybox', role='rootfs', tag='installed', domain="arm.helloworld")
Label('package', 'busybox', role='rootfs', tag='installed', domain='arm.helloworld')
>>> str(_)
'package:(arm.helloworld)busybox{rootfs}/installed'
>>> Label('package', 'busybox', role='rootfs', tag='installed', domain="arm(helloworld)")
Label('package', 'busybox', role='rootfs', tag='installed', domain='arm(helloworld)')
>>> str(_)
'package:(arm(helloworld))busybox{rootfs}/installed'

FLAG_DOMAIN_SWEEP = 'D'

FLAG_SYSTEM = 'S'

FLAG_TRANSIENT = 'T'

copy()

Return a copy of this label.

copy_and_unify_with(target)

Return a copy of ourserlves, unified with the target.

All the non-wildcard parts of ‘target’ are copied, to overwrite the equivalent parts of the new label.

copy_with_domain(new_domain)

Return a copy of self, with the domain changed to new_domain.

Note that if ‘new_domain’ is given as “” (empty string) then that is treated as if it were given as None.

copy_with_role(new_role)

Return a copy of self, with the role changed to new_role.

copy_with_tag(new_tag, system=None, transient=None)

Return a copy of self, with the tag changed to new_tag.

domain

domain_part = '[()A-Za-z0-9._+-]+|\\*'

domain_part_re = <_sre.SRE_Pattern object>

fragment_re = <_sre.SRE_Pattern object at 0x28e60e0>

static from_fragment(fragment, default_type, default_role=None, default_domain=None)

Given a string containing a label fragment, return a Label.

The caller indicates the default type, role and domain.

The fragment must contain a <name>, but otherwise may contain any of:

• <type>: - if this is not given, the default is used
• (<domain>) - if this is not given, the default is used.
• {<role>} - if this is not given, the default is used.
• /<tag> - if this is not given, a tag appropriate to the <type> is chosen (checked_out, postinstalled or deployed)

Any of the default_xx values may be None.

static from_string(label_string)

Construct a Label from its string representation.

The string should be of the correct form:

• <type>:<name>/<tag>
• <type>:<name>{<role>}/<tag>
• <type>:<name>/<tag>[<flags>]
• <type>:<name>{<role>}/<tag>[<flags>]
• <type>:(<domain>)<name>/<tag>
• <type>:(<domain>)<name>{<role>}/<tag>
• <type>:(<domain>)<name>/<tag>[<flags>]
• <type>:(<domain>)<name>{<role>}/<tag>[<flags>]

See the docstring for Label itself for the meaning of the various parts of a label.

<flags> is a set of individual characters indicated as flags. There are two flags that will be recognised and used, ‘T’ for Transience and ‘S’ for System. Any other flag characters will be ignored.

If the label string is valid, a corresponding Label will be returned, otherwise a utils.GiveUp exception will be raised.

>>> Label.from_string('package:busybox/installed')
Label('package', 'busybox', role=None, tag='installed')
>>> Label.from_string('package:busybox{firmware}/installed[ABT]')
Label('package', 'busybox', role='firmware', tag='installed', transient=True)
>>> Label.from_string('package:(arm.hello)busybox{firmware}/installed[ABT]')
Label('package', 'busybox', role='firmware', tag='installed', transient=True, domain='arm.hello')
>>> Label.from_string('*:(*)*{*}/*')
Label('*', '*', role='*', tag='*', domain='*')
>>> Label.from_string('*:*{*}/*')
Label('*', '*', role='*', tag='*')
>>> Label.from_string('foo:bar{baz}/wombat[T]')
Label('foo', 'bar', role='baz', tag='wombat', transient=True)
>>> Label.from_string('foo:(ick)bar{baz}/wombat[T]')
Label('foo', 'bar', role='baz', tag='wombat', transient=True, domain='ick')
>>> Label.from_string('foo:(ick(ack))bar{baz}/wombat[T]')
Label('foo', 'bar', role='baz', tag='wombat', transient=True, domain='ick(ack)')

A tag must be supplied:

>>> Label.from_string('package:busybox')
Traceback (most recent call last):
...
GiveUp: Label string 'package:busybox' is not a valid Label

If you specify a domain, it may not be “empty”:

>>> Label.from_string('package:()busybox/*')
Traceback (most recent call last):
...
GiveUp: Label string 'package:()busybox/*' is not a valid Label

is_definite()

Return True iff this label contains no wildcards

is_wildcard()

Return True iff this label contains at least one wildcard.

This is the dual of is_definite(), but is provided so whichever seems more appropriate to the task at hand can be chosen.

just_match(other)

Return True if the labels match, False if they do not

label_part = '[A-Za-z0-9._+-]+|\\*'

label_part_re = <_sre.SRE_Pattern object>

label_string_re = <_sre.SRE_Pattern object at 0x266a770>

match(other)

Return an integer indicating the match specicifity - which we do by counting ‘*’ s and subtracting from 0.

Returns the match specicifity, None if there wasn’t one.

match_without_tag(other)

Returns True if other matches self without the tag, False otherwise

Specifically, tests whether the two Labels have identical type, domain, name and role.

middle()

Return the “middle” portion of our name, between type and tag.

That is, the domain, name and role (as appropriate).

For instance:

>>> checkout('fred').middle()
'fred'
>>> checkout('jim', domain='a(b)', tag='*').middle()
'(a(b))jim'
>>> package('fred', role='bob', domain='a').middle()
'(a)fred{bob}'

name

rehash()

Calculate the hash for a label.

Ignores the domain name (since that may be changed) and the transient and system flags (since they are defined to be, well, transient).

role

static split_domain(value)

Split a domain into its parts and check that it is valid.

Note that the ‘value’ should not include the outermost parentheses (see the examples below).

Raises a utils.GiveUp exception if it’s Bad.

For instance:

>>> Label.split_domain('fred')
['fred']
>>> Label.split_domain('fred(jim)')
['fred', 'jim']
>>> Label.split_domain('fred(jim(bob))')
['fred', 'jim', 'bob']
>>> Label.split_domain('')
Traceback (most recent call last):
...
GiveUp: Label domain '()' is not allowed
>>> Label.split_domain('()')
Traceback (most recent call last):
...
GiveUp: Label domain '(())' starts with zero length domain, '(()', i.e. '(('
>>> Label.split_domain('(')
Traceback (most recent call last):
...
GiveUp: Label domain part '(()' has unbalanced parentheses, '('
>>> Label.split_domain(')')
Traceback (most recent call last):
...
GiveUp: Label domain '())' has unbalanced parentheses, ')'
>>> Label.split_domain('fred(jim')
Traceback (most recent call last):
...
GiveUp: Label domain part '(fred(jim)' has unbalanced parentheses, 'fred(jim'
>>> Label.split_domain('fred((jim(bob)))')
Traceback (most recent call last):
...
GiveUp: Label domain '(fred((jim(bob))))' starts with zero length domain, '((jim(bob))', i.e. '(('

split_domains()

Returns a list of the domains for this Label, in order.

If there are no subdomains, then a zero length list is returned.

Raises a utils.GiveUp exception if the parentheses do not match up (the check is only fairly crude), or if there are two adjacent opening parentheses.

This is similar to utils.split_domain, but behaves somewhat differently.

tag

type

unifies(other)

Returns True if and only if every field in self is either equal to a field in other , or if other is a wildcard. Wildcards in self do not match anything but a wildcard in other.

class muddled.depend.Rule(target_dep, action)

Bases: object

A rule or “dependency set”.

Every Rule has:

• a target Label (its desired result),
• an optional Action object (to do the work to produce that result),
• and a set of Labels on which the target depends (which must have been satisfied before this Rule can be triggered).

In other words, once all the dependency Labels are satisfied, the object can be called to ‘build’ the target Label.

(And if there is no object, the target is automatically satisfied.)

Note that the “set of Labels” is indeed a set, so adding the same Label more than once will not have any effect (caveat: adding a label with different flags from a previous label may have an effect, but it’s not something that should be relied on).

Note

The actual “satisfying” of labels is done in muddled.mechanics. For instance, Builder.build_label() “builds” a label in the context of the rest of its environment, and uses ‘action’ to “build” the label.

• target_dep is the Label this Rule intends to “make”.
• action is None or an Action, which will be used to “make” the target_dep.

add(label)

Add a dependency on the given Label.

catenate_and_merge(other_rule, complainOnDuplicate=False, replaceOnDuplicate=True)

Merge ourselves with the given rule.

If replaceOnDuplicate is true, other_rule get priority - this is the target for a unify() and makes the source build instructions go away.

depend_checkout(co_name, tag)

Add a dependency on label “checkout:<co_name>/<tag>”.

depend_deploy(dep_name, tag)

Add a dependency on label “deployment:<dep_name>/tag”.

depend_pkg(pkg, role, tag)

Add a dependency on label “package:<pkg>{<role>}/tag”.

merge(deps)

Merge another Rule with this one.

Adds all the dependency labels from deps to this Rule.

If deps.action is not None, replaces our action with the one from deps.

replace_target(new_t)

to_string(showSystem=True, showUser=True)

Return a string representing this dependency set.

If showSystem is true, include dependency labels with the System tag (i.e., dependencies inserted by the muddle system itself), otherwise ignore such.

If showUser is true, include dependency labels without the System tag (i.e., “normal” dependencies, explicitly added by the user), otherwise ignore such.

The default is to show all of the dependencies.

For instance (not a very realistic example):

>>> tgt = Label.from_string('package:fred{jim}/*')
>>> r = Rule(tgt,None)
>>> r.to_string()
'package:fred{jim}/* <- [ ]'
>>> r.add(Label.from_string('package:bob{bob}/built'))
>>> r.depend_checkout('fred','jim')
>>> r.depend_pkg('albert','jim','built')
>>> r.depend_deploy('hall','deployed')
>>> r.to_string()
'package:fred{jim}/* <- [ checkout:fred/jim, deployment:hall/deployed, package:albert{jim}/built, package:bob{bob}/built ]'

The “<-” is to be read “depends on”.

Note that the order of the dependencies in the output is sorted by label.

unify_dependencies(source, target)

Whenever source appears in our dependencies, replace it with source.unify(target)

class muddled.depend.RuleSet

Bases: object

A collection of rules that encapsulate how you can get from A to B.

Formally, this is just a mapping of labels to Rules. Duplicate targets are merged - it’s assumed that the objects will be the same.

Informally, we cache some of the label look-ups for a major improvement in build time; the half billion lookups were taking over a hundred seconds to do!

CAVEAT: Be aware that new rules (when added) can be merged into existing rules. Since we don’t copy rules when we add them, this could be a cause of unexpected side effects...

add(rule)

Add the Rule ‘rule’.

Specifically, if we already have a rule for this rule’s target label, merge the new rule into the old (see Rule.merge).

If this rule is for a new target, just remember it.

merge(other_deps)

Merge another RuleSet into this one.

Simply adds each rule from the other RuleSet to this one (see the ‘RuleSet.add’ method)

rule_for_target(target, createIfNotPresent=False)

Return the rule for this target - this contains all the labels that need to be asserted in order to build the target.

If createIfNotPresent is true, and there is no rule for this target, then we will create (and add to our internal map) an empty Rule for this target.

Otherwise, if there is no rule for this target, we return None

rules_for_target(label, useTags=True, useMatch=True)

Return the set of rules for any target(s) matching the given label.

• If useTags is true, then we should take account of tags when matching, otherwise we should ignore them. If useMatch is true, then useTags is ignored.
• If useMatch is true, then we allow wildcards in ‘label’, otherwise we do not.

Returns the set of Rules found, or an empty set if none were found.

rules_which_depend_on(label, useTags=True, useMatch=True)

Given a label, return a set of the rules which have it as one of their dependencies.

If there are no rules which have this label as one of their dependencies, we return the empty set.

• If useTags is true, then we should take account of tags when matching, otherwise we should ignore them. If useMatch is true, then useTags is ignored.
• If useMatch is true, then we allow wildcards in ‘label’, otherwise we do not.

targets_match(target, useMatch=True)

Return the set of targets matching the given ‘target’ label.

If useMatch is true, allow wildcards in ‘target’ (in which case more than one result may be obtained). If useMatch is false, then at most one match can be found (‘target’ itself).

Returns a set of suitable targets, or an empty set if there are none.

to_string(matchLabel=None, showUser=True, showSystem=True, ignore_empty=False)

Return a string representing this rule set.

If showSystem is true, include dependency labels with the System tag (i.e., dependencies inserted by the muddle system itself), otherwise ignore such.

If showUser is true, include dependency labels without the System tag (i.e., “normal” dependencies, explicitly added by the user), otherwise ignore such.

The default is to show all of the dependencies.

For instance (not a very realistic example):

>>> l = Label.from_string('package:fred{bob}/initial')
>>> r = RuleSet()
>>> depend_chain(None, l, ['built', 'bamboozled'], r)
>>> print str(r)
-----
package:fred{bob}/bamboozled <- [ package:fred{bob}/built ]
package:fred{bob}/built <- [ package:fred{bob}/initial ]
package:fred{bob}/initial <- [ ]
-----

The “<-” is to be read “depends on”.

Note that the order of the rules in the output is sorted by target label, and is thus reproducible.

unify(source, target)

Merge source into target.

This is a pain, and depends heavily on CatenatedObject

wrap_actions(generator, label)

class muddled.depend.SequentialAction(a, b)

Bases: object

Invoke two actions in turn

build_label(builder, label)

muddled.depend.checkout(name, tag='*', domain=None)

A simple convenience function to return a checkout label

• ‘name’ is the checkout name
• ‘tag’ is the label tag, defaulting to “*”
• ‘domain’ is the label domain name, defaulting to None

For instance:

>>> l = checkout('fred')
>>> l == Label.from_string('checkout:fred/*')
True

muddled.depend.depend_chain(action, label, tags, ruleset)

Add a chain of dependencies to the given ruleset.

This is perhaps best explained with an example:

>>> l = Label.from_string('package:fred{bob}/initial')
>>> r = RuleSet()
>>> depend_chain(None, l, ['built', 'bamboozled'], r)
>>> print str(r)
-----
package:fred{bob}/bamboozled <- [ package:fred{bob}/built ]
package:fred{bob}/built <- [ package:fred{bob}/initial ]
package:fred{bob}/initial <- [ ]
-----

muddled.depend.depend_empty(action, label)

Create a dependency set with no prerequisites - simply signals that a tag is available to be built at any time.

muddled.depend.depend_none(action, label)

Quick rule that makes label depend on nothing.

muddled.depend.depend_one(action, label, dep_label)

Quick rule that makes label depend only on dep_label.

muddled.depend.depend_self(action, label, old_tag)

Make a quick dependency set that depends just on you. Used by some of the standard package and checkout classes to quickly build standard dependency sets.

muddled.depend.deployment(name, tag='*', domain=None)

A simple convenience function to return a deployment label

• ‘name’ is the deployment name
• ‘tag’ is the label tag, defaulting to “*”
• ‘domain’ is the label domain name, defaulting to None

For instance:

>>> l = deployment('fred')
>>> l == Label.from_string('deployment:fred/*')
True

muddled.depend.label_from_string(str)

Do not use this!!! Can you say “deprecated”?

This function was originally removed, since it is replaced by Label.from_string. However, too many old builds still attempt to import it, which can cause problems at the “muddle init” stage, and also with “muddle unstamp”.

Please do not use this function in new builds.

muddled.depend.label_list_to_string(labels, join_with=' ')

muddled.depend.label_set_to_string(label_set, start_with='[', end_with=']', join_with=', ')

Utility function to convert a label set to a string.

muddled.depend.needed_to_build(ruleset, target, useTags=True, useMatch=False)

Given a rule set and a target, return a complete list of the rules needed to build the target.

• If useTags is true, then we should take account of tags when looking for the rules for this ‘target’, otherwise we should ignore them.
• If useMatch is true, then we allow wildcards in ‘target’, otherwise we do not.

Returns a list of rules.

muddled.depend.normalise_checkout_label(label, tag='*')

Given a checkout label with random “other” fields, normalise it.

Returns a normalised checkout label, with the role unset and the tag set to ‘tag’ (normally, “*”). This may be the same label (if it was already normalised), or it may be a new label. No guarantee is given of either.

Raise a MuddleBug exception if the label is not a checkout label.

A normalised checkout label:

1. Has the given tag (normally ‘*’, sometimes LabelTag.CheckedOut)
2. Does not have a role (checkout labels do not use the role)
3. Does not have the system or transient flags set
4. Has the same name and (if present) domain

muddled.depend.package(name, role, tag='*', domain=None)

A simple convenience function to return a package label

• ‘name’ is the package name
• ‘role’ is the package role
• ‘tag’ is the label tag, defaulting to “*”
• ‘domain’ is the label domain name, defaulting to None

For instance:

>>> l = package('fred', 'jim')
>>> l == Label.from_string('package:fred{jim}/*')
True

muddled.depend.required_by(ruleset, label, useTags=True, useMatch=True)

Given a ruleset and a label, form the list of labels that (directly or indirectly) depend on label. We deliberately do not give you the associated rules since you will want to call needed_to_build() individually to ensure that other prerequisites are satisfied.

The order in which we give you the labels gives you a hint as to a logical order to rebuild in (i.e. one the user will vaguely understand).

• useMatch - If True, do wildcard matches, else do an exact comparison.

• useTags - If False, we discount the value of a tag - this effectively

  results in a wildcard tag search.

Returns a set of labels to build.

muddled.depend.retag_label_list(labels, new_tag)

Does what it says on the tin, returning the new label list.

That is, returns a list formed by copying each Label in ‘labels’ and setting its tag to the given ‘new_tag’.

muddled.depend.rule_list_to_string(rule_list)

Utility function to convert a rule list to a string.

muddled.depend.rule_target_str(rule)

Take a rule and return its target as a string. Mainly used as an argument for map so we can print lists of rules sensibly.

muddled.depend.rule_with_least_dependencies(rules)

Given a (Python) set of rules, find the ‘best’ one to use.

This is actually impossible by any rational metric, so you usually only expect to call this function with a set of size 1, in which case our metric really doesn’t matter.

However, in a vague attempt to be somewhat intelligent, we return the element with the fewest direct dependencies.

muddled.utils

Muddle utilities.

class muddled.utils.Choice(choices)

Bases: object

A choice “sequence”.

A choice sequence is:

• a string or dictionary, the only choice. For instance:

  choice = Choice("libxml-dev2")
  assert choice.choose('any string at all') == 'libxml-dev2)
  

• a sequence of the form [ (pattern, value), ... ]; that is a sequence of one or more ‘(pattern, value)’ pairs, where each ‘pattern’ is an fnmatch pattern (see below) and each ‘value’ is a string or dict.

  The patterns are compared to ‘what_to_match’ in turn, and if one matches, the corresponding ‘value’ is returned. If none match, a ValueError is raised.

  For instance:

  choice = Choice([ ('ubuntu-12.*', 'package-v12'),
                    ('ubuntu-1?.*', 'package-v10') ])
  try:
      match = choice.choose_to_match_os()
  except ValueError:
      print 'No package matched OS %s'%get_os_version_name()
  

• a sequence of the form [ (pattern, value), ..., default ]; that is a sequence of one or more pairs (as above), with a final “default” value, which must be a string or dict or None.

  The patterns are compared to ‘what_to_match’ in turn, and if one matches, the corresponding ‘value’ is returned. If none match, the final default value is returned. None is allowed so the caller can easily tell that no choice was actually made.

  choice = Choice([ (‘ubuntu-12.*’, ‘package-v12’),

  (‘ubuntu-1?.*’, ‘package-v10’), ‘package-v09’ ])

  System Message: WARNING/2 (/home/docs/checkouts/readthedocs.org/user_builds/muddle/checkouts/latest/muddled/utils.py:docstring of muddled.utils.Choice, line 39)

  Definition list ends without a blank line; unexpected unindent.

  # ‘match’ will always have a “sensible” value match = choice.choose_to_match_os()

  choice = Choice([ (‘ubuntu-12.*’, ‘package-v12’),

  (‘ubuntu-1?.*’, ‘package-v10’), None ])

  System Message: WARNING/2 (/home/docs/checkouts/readthedocs.org/user_builds/muddle/checkouts/latest/muddled/utils.py:docstring of muddled.utils.Choice, line 45)

  Definition list ends without a blank line; unexpected unindent.

  match = choice.choose_to_match_os() if match is None:

  System Message: ERROR/3 (/home/docs/checkouts/readthedocs.org/user_builds/muddle/checkouts/latest/muddled/utils.py:docstring of muddled.utils.Choice, line 47)

  Unexpected indentation.

  return # We know there was no given value

• as a result of the previous, we also allow [default], although [None] is of questionable utility.

  choice = Choice([“libxml-dev2”]) assert choice.choose(‘any string at all’) == ‘libxml-dev2)

  choice = Choice([“None”]) assert choice.choose(‘any string at all’) is None

  (although that latter is the only way of “forcing” a Choice that will always return None, if you did need such a thing...)

Why not just use a list of pairs (possibly with a default string at the end, essentially just what we pass to Choice)? Well, it turns out that if you want to do something like:

pkgs.apt_get(["fromble1",
              Choice([ ('ubuntu-12.*', 'fromble'),
                       ('ubuntu-11.*', 'alex'),
                       None ]),
              "ribbit",
              Choice([ ('ubuntu-12.*', 'package-v12'),
                       ('ubuntu-1?.*', 'package-v10'),
                       'package-v7' ]),
              "libxml-dev2",
            ])

it is (a) really hard to type it right if it is just nested sequences, and (b) terribly hard to give useful error messages when the user doesn’t get it right. There are already enough brackets of various sorts, and if we don’t have the “Choice” delimiters, it just gets harder to keep track.

choose(what_to_match)

Try to match ‘what_to_match’, and return the appropriate value.

Raises ValueError if there is no match.

Returns None if (and only if) that was given as a fall-back default value.

choose_to_match_os(version_name=None)

A special case of ‘decide’ to match OS id/version

If ‘version_name’ is None, then it looks up the system ‘id’ (the “name” of the OS, e.g., “ubuntu”), and ‘version’ of the OS (e.g., “12.10”) from /etc/os-release, and concatenates them separated by a space (so “ubuntu 12.10”).

It returns the result of calling:

choose(version_name)

So, on an Ubuntu system (which also includes a Linux Mint system, since its /etc/os-release identifies it as the underlying Ubuntu system), one might do:

choice = Choice([ (‘ubuntu-12.*’, ‘package-v12’),

(‘ubuntu-1?.*’, ‘package-v10’), ‘package-v7’ ])

System Message: WARNING/2 (/home/docs/checkouts/readthedocs.org/user_builds/muddle/checkouts/latest/muddled/utils.py:docstring of muddled.utils.Choice.choose_to_match_os, line 19)

Definition list ends without a blank line; unexpected unindent.

choice.choose_to_match_os()

to choose the appropriate version of “package” depending on the OS.

muddled.utils.Error

alias of MuddleBug

muddled.utils.Failure

alias of GiveUp

exception muddled.utils.GiveUp(message=None, retcode=1)

Bases: exceptions.Exception

Use this to indicate that something has gone wrong and we are giving up.

This is not an error in muddle itself, however, so there is no need for a traceback.

By default, a return code of 1 is indicated by the ‘retcode’ value - this can be set by the caller to another value, which __main__.py should then use as its return code if the exception reaches it.

retcode = 1

class muddled.utils.HashFile(name, mode='r', ignore_comments=False, ignore_blank_lines=False)

Bases: object

A very simple class for handling files and calculating their SHA1 hash.

We support a subset of the normal file class, but as lines are read or written, we calculate a SHA1 hash for the file.

Optionally, comment lines and/or blank lines can be ignored in calculating the hash, where comment lines are those starting with a ‘#’, or whitespace and a ‘#’, and blank lines are those which contain only whitespace (which includes empty lines).

Open the file, for read or write.

• ‘name’ is the name (path) of the file to open
• ‘mode’ is either ‘r’ (for read) or ‘w’ (for write). If ‘w’ is specified, then if the file doesn’t exist, it will be created, otherwise it will be truncated.
• if ‘ignore_comments’ is true, then lines starting with a ‘#’ (or whitespace and a ‘#’) will not be used in calculating the hash.
• if ‘ignore_blank_lines’ is true, then lines that are empty (zero length), or contain only whitespace, will not be used in calculating the hash.

Note that this “ignore” doesn’t mean “don’t write to the file”, it just means “ignore when calculating the hash”.

close()

Close the file.

hash()

Return the SHA1 hash, calculated from the lines so far, as a hex string.

next()

readline()

Read the next line from the file, and add it to the SHA1 hash as well.

Returns ‘’ if there is no next line (i.e., EOF is reached).

write(text)

Write the give text to the file, and add it to the SHA1 hash as well.

(Unless we are ignoring comment lines and it is a comment line, or we are ignoring blank lines and it is a blank line, in which case it will be written to the file but not added to the hash.)

As is normal for file writes, the ‘n’ at the end of a line must be specified.

exception muddled.utils.MuddleBug(message=None, retcode=1)

Bases: muddled.utils.GiveUp

Use this to indicate that something has gone wrong with muddle itself.

We thus expect that a traceback will be produced.

class muddled.utils.MuddleOrderedDict

Bases: _abcoll.MutableMapping

A simple dictionary-like class that returns keys in order of (first) insertion.

class muddled.utils.MuddleSortedDict

Bases: _abcoll.MutableMapping

A simple dictionary-like class that returns keys in sorted order.

exception muddled.utils.ShellError(cmd, retcode, output=None)

Bases: muddled.utils.GiveUp

exception muddled.utils.Unsupported(message=None, retcode=1)

Bases: muddled.utils.GiveUp

Use this to indicate that an action is unsupported.

This is used, for instance, when git reports that it will not pull to a shallow clone, which is not an error, but the user will want to know.

This is deliberately a subclass of GiveUp, because it is telling muddle to give up an operation.

class muddled.utils.VersionNumber(major=0, minor=0)

Bases: object

Simple support for two part “semantic version” numbers.

Such version numbers are of the form <major>.<minor>

static from_string(s)

next()

Return the next (minor) version number.

static unset()

Return an unset version number.

Unset version numbers compare less than proper ones.

muddled.utils.arch_name()

Retrieve the name of the architecture on which we’re running. Some builds require packages to be built on a particular (odd) architecture.

muddled.utils.c_escape(v)

Escape sensitive characters in v.

muddled.utils.calc_file_hash(filename)

Calculate and return the SHA1 hash for the named file.

muddled.utils.copy_file(from_path, to_path, object_exactly=False, preserve=False, force=False)

Copy a file (either a “proper” file, not a directory, or a symbolic link).

Just like recursively_copy, only not recursive :-)

If the target file already exists, it is overwritten.

Caveat: if the target file is a directory, it will not be overwritten. If the source file is a link, being copied as a link, and the target file is not a link, it will not be overwritten.

If ‘object_exactly’ is true, then if ‘from_path’ is a symbolic link, it will be copied as a link, otherwise the referenced file will be copied.

If ‘preserve’ is true, then the file’s mode, ownership and timestamp will be copied, if possible. Note that on Un*x file ownership can only be copied if the process is running as ‘root’ (or within ‘sudo’).

If ‘force’ is true, then if a target file is not writeable, try removing it and then copying it.

muddled.utils.copy_file_metadata(from_path, to_path)

Copy file metadata.

If ‘to_path’ is a link, then it tries to copy whatever it can from ‘from_path’, treated as a link.

If ‘to_path’ is not a link, then it copies from ‘from_path’, or, if ‘from_path’ is a link, whatever ‘from_path’ references.

Metadata is: mode bits, atime, mtime, flags and (if the process has an effective UID of 0) the ownership (uid and gid).

muddled.utils.copy_name_list_with_dirs(file_list, old_root, new_root, object_exactly=True, preserve=False)

Given file_list, create file_list[new_root/old_root], creating any directories you need on the way.

file_list is a list of full path names. old_root is the old root directory new_root is where we want them copied

muddled.utils.copy_without(src, dst, without=None, object_exactly=True, preserve=False, force=False, verbose=True)

Copy files from the ‘src’ directory to the ‘dst’ directory, without those in ‘without’

If given, ‘without’ should be a sequence of filenames - for instance, [‘.bzr’, ‘.svn’].

If ‘object_exactly’ is true, then symbolic links will be copied as links, otherwise the referenced file will be copied.

If ‘preserve’ is true, then the file’s mode, ownership and timestamp will be copied, if possible. Note that on Un*x file ownership can only be copied if the process is running as ‘root’ (or within ‘sudo’).

If ‘force’ is true, then if a target file is not writeable, try removing it and then copying it.

If ‘verbose’ is true (the default), print out what we’re copying.

Creates directories in the destination, if necessary.

Uses copy_file() to copy each file.

muddled.utils.current_machine_name()

Return the identity of the current machine - possibly including the domain name, possibly not

muddled.utils.current_user()

Return the identity of the current user, as an email address if possible, but otherwise as a UNIX uid

muddled.utils.debian_version_is(test, ref)

Return 1 if test > ref, -1 if ref > test, 0 if they are equal

muddled.utils.do_shell_quote(str)

muddled.utils.domain_subpath(domain_name)

Calculate the sub-path for a given domain name.

For instance:

>>> domain_subpath('a')
'domains/a'
>>> domain_subpath('a(b)')
'domains/a/domains/b'
>>> domain_subpath('a(b(c))')
'domains/a/domains/b/domains/c'
>>> domain_subpath('a(b(c)')
Traceback (most recent call last):
...
GiveUp: Domain name "a(b(c)" has mis-matched parentheses

muddled.utils.dynamic_load(filename)

muddled.utils.ensure_dir(dir, verbose=True)

Ensure that dir exists and is a directory, or throw an error.

muddled.utils.find_by_predicate(source_dir, accept_fn, links_are_symbolic=True)

Given a source directory and an acceptance function

fn(source_base, file_name) -> result

Obtain a list of [result] if result is not None.

muddled.utils.find_domain(root_dir, dir)

Find the domain of ‘dir’.

‘root_dir’ is the root of the (entire) muddle build tree.

This function basically works backwards through the path of ‘dir’, until it reaches ‘root_dir’. As it goes, it assembles the full domain name for the domain enclosing ‘dir’.

Returns the domain name, or None if ‘dir’ is not within a subdomain, and the directory of the root of the domain. That is:

(domain_name, domain_dir) or (None, None)

muddled.utils.find_label_dir(builder, label)

Given a label, find the corresponding directory.

• for checkout labels, the checkout directory
• for package labels, the install directory
• for deployment labels, the deployment directory

This is the heart of “muddle query dir”.

muddled.utils.find_local_relative_root(builder, label)

Given a label, find its “local” root directory, relative to toplevel.

Calls find_local_root() and then calculates the location of that relative to the root of the entire muddle build tree.

muddled.utils.find_local_root(builder, label)

Given a label, find its “local” root directory.

For a normal label, this will be the normal muddle root directory (where the top-level .muddle/ directory is).

For a label in a subdomain, it will be the root directory of that subdomain - again, where its .muddle/ directory is.

muddled.utils.find_root_and_domain(dir)

Find the build tree root containing ‘dir’, and find the domain of ‘dir’.

This function basically works backwards through the path of ‘dir’, until it finds a directory containing a ‘.muddle/’ directory, that is not within a subdomain. As it goes, it assembles the full domain name for the domain enclosing ‘dir’.

Returns a pair (root_dir, current_domain).

If ‘dir’ is not within a subdomain, then ‘current_domain’ will be None.

If ‘dir’ is not within a muddle build tree, then ‘root_dir’ will also be None.

muddled.utils.get_cmd_data(thing, env=None, show_command=False)

Run the command ‘thing’, and return its output.

‘thing’ may be a string (e.g., “ls -l”) or a sequence (e.g., [“ls”, “-l”]). Internally, a string will be converted into a sequence before it is used. Any non-string items in a ‘thing’ sequence will be converted to strings using ‘str()’ (e.g., if a Label instance is given).

If ‘env’ is given, then it is the environment to use when running ‘thing’, otherwise ‘os.environ’ is used.

Note that the output of the command is not shown whilst the command is running.

If the command returns a non-zero exit code, then we raise a ShellError.

(This is basically a muddle-flavoured wrapper around subprocess.check_output)

muddled.utils.get_domain_name_from(dir)

Given a directory ‘dir’, extract the domain name.

‘dir’ should not end with a trailing slash.

It is assumed that ‘dir’ is of the form “<something>/domains/<domain_name>”, and we want to return <domain_name>.

muddled.utils.get_os_version_name()

Retrieve a string identifying this version of the operating system

Looks in /etc/os-release, which gives a different result than platform.py, which looks in /etc/lsb-release.

muddled.utils.get_prefix_pair(prefix_one, value_one, prefix_two, value_two)

Returns a pair (prefix_onevalue_one, prefix_twovalue_two) - used by rrw.py as a utility function

muddled.utils.indent(text, indent)

Return the text indented with the ‘indent’ string.

(i.e., place ‘indent’ in front of each line of text).

muddled.utils.is_release_build(dir)

Check if the given ‘dir’ is the top level of a release build.

‘dir’ should be the path to the directory contining the build’s .muddle directory (the “top” of the build).

The build is assumed to be a release build if there is a file called .muddle/Release.

muddled.utils.is_subdomain(dir)

Check if the given ‘dir’ is a (sub)domain.

‘dir’ should be the path to the directory contining the build’s .muddle directory (the “top” of the build).

The build is assumed to be a (sub)domain if there is a file called .muddle/am_subdomain.

muddled.utils.iso_time()

Retrieve the current time and date in ISO style YYYY-MM-DD HH:MM:SS.

muddled.utils.join_domain(domain_parts)

Re-join a domain name we split with split_domain.

muddled.utils.mark_as_domain(dir, domain_name)

Mark the build in ‘dir’ as a (sub)domain

This is done by creating a file .muddle/am_subdomain

‘dir’ should be the path to the directory contining the sub-build’s .muddle directory (the “top” of the sub-build).

‘dir’ should thus be of the form “<somewhere>/domains/<domain_name>”, but we do not check this.

The given ‘domain_name’ is written to the file, but this should not be particularly trusted - refer to the containing directory structure for the canonical domain name.

muddled.utils.maybe_shell_quote(str, doQuote)

If doQuote is False, do nothing, else shell-quote str.

Annoyingly, shell quoting things correctly must use backslashes, since quotes can (and will) be misinterpreted. Bah.

NB: Despite the name, this is actually “escaping”, rather then “quoting”. Specifically, any single quote, double quote or backslash characters in the original string will be converted to a backslash followed by the original character, in the final string.

muddled.utils.normalise_dir(dir)

muddled.utils.normalise_path(dir)

muddled.utils.num_cols()

How many columns on our terminal?

If it can’t tell (e.g., because it curses is not available), returns 70.

muddled.utils.pad_to(str, val, pad_with=' ')

Pad the given string to the given number of characters with the given string.

muddled.utils.page_text(progname, text)

Try paging ‘text’ by piping it through ‘progname’.

Looks for ‘progname’ on the PATH, and if os.environ[‘PATH’] doesn’t exist, tries looking for it on os.defpath.

If an executable version of ‘progname’ can’t be found, just prints the text out.

If ‘progname’ is None (or an empty string, or otherwise false), then just print ‘text’.

muddled.utils.parse_etc_os_release()

Parse /etc/os-release and return a dictionary

This is not a good parser by any means - it is the quickest and simplest thing I could do.

Note that a line like:

FRED='Fred's name'

will give us:

key 'Fred' -> value r"Fred's name"

i.e., we do not treat backslashes in a string in any way at all. In fact, we don’t do anything with strings other than throw away paired outer “’” or ‘”’.

Oh, also we don’t check whether the names before the ‘=’ signs are those that are expected, although we do provide a default value for ‘ID’ if it is not given (as the documentation for /etc/os-release’ specified).

(Note that the standard library platform.py (in 2.7) looks at /etc/lsb-release, instead of /etc/os-release, which gives different results.)

muddled.utils.parse_gid(builder, text_gid)

Todo

One day, we should do something more intelligent than just assuming your gid is numeric

muddled.utils.parse_mode(in_mode)

Parse a UNIX mode specification into a pair (clear_bits, set_bits).

muddled.utils.parse_uid(builder, text_uid)

Todo

One day, we should do something more intelligent than just assuming your uid is numeric

muddled.utils.print_string_set(ss)

Given a string set, return a string representing it.

muddled.utils.quote_list(lst)

Given a list, quote each element of it and return them, space separated

muddled.utils.recursively_copy(from_dir, to_dir, object_exactly=False, preserve=True, force=False)

Take everything in from_dir and copy it to to_dir, overwriting anything that might already be there.

Dot files are included in the copying.

If object_exactly is true, then symbolic links will be copied as links, otherwise the referenced file will be copied.

If preserve is true, then the file’s mode, ownership and timestamp will be copied, if possible. This is only really useful when copying as a privileged user.

If ‘force’ is true, then if a target file is not writeable, try removing it and then copying it.

muddled.utils.recursively_remove(a_dir)

Recursively demove a directory.

muddled.utils.rel_join(vroot, path)

Find what path would be called if it existed inside vroot. Differs from os.path.join() in that if path contains a leading ‘/’, it is not assumed to override vroot.

If vroot is none, we just return path.

muddled.utils.replace_root_name(base, replacement, filename)

Given a filename, a base and a replacement, replace base with replacement at the start of filename.

muddled.utils.run0(thing, env=None, show_command=True, show_output=True)

Run the command ‘thing’, returning nothing.

(Run and return 0 values)

‘thing’ may be a string (e.g., “ls -l”) or a sequence (e.g., [“ls”, “-l”]). Internally, a string will be converted into a sequence before it is used.

If ‘env’ is given, then it is the environment to use when running ‘thing’, otherwise ‘os.environ’ is used.

If ‘show_command’ is true, then “> <thing>” will be printed out before running the command.

If ‘show_output’ is true, then the output of the command (both stdout and stderr) will be printed out as the command runs. Note that this is the default.

If the command returns a non-zero return code, then a ShellError will be raised, containing the returncode, the command string and any output that occurred.

muddled.utils.run1(thing, env=None, show_command=True, show_output=False)

Run the command ‘thing’, returning its output.

(Run and return 1 value)

‘thing’ may be a string (e.g., “ls -l”) or a sequence (e.g., [“ls”, “-l”]). Internally, a string will be converted into a sequence before it is used. Any non-string items in a ‘thing’ sequence will be converted to strings using ‘str()’ (e.g., if a Label instance is given).

If ‘env’ is given, then it is the environment to use when running ‘thing’, otherwise ‘os.environ’ is used.

If ‘show_command’ is true, then “> <thing>” will be printed out before running the command.

If ‘show_output’ is true, then the output of the command (both stdout and stderr) will be printed out as the command runs.

If the command returns a non-zero return code, then a ShellError will be raised, containing the returncode, the command string and any output that occurred.

Otherwise, the command output (stdout and stderr combined) is returned.

muddled.utils.run2(thing, env=None, show_command=True, show_output=False)

Run the command ‘thing’, returning the return code and output.

(Run and return 2 values)

‘thing’ may be a string (e.g., “ls -l”) or a sequence (e.g., [“ls”, “-l”]). Internally, a string will be converted into a sequence before it is used. Any non-string items in a ‘thing’ sequence will be converted to strings using ‘str()’ (e.g., if a Label instance is given).

If ‘show_command’ is true, then “> <thing>” will be printed out before running the command.

If ‘show_output’ is true, then the output of the command (both stdout and stderr) will be printed out as the command runs.

The output of the command (stdout and stderr) goes to the normal stdout whilst the command is running.

The command return code and output are returned as a tuple:

(retcode, output)

muddled.utils.run3(thing, env=None, show_command=True, show_output=False)

Run the command ‘thing’, returning the return code, stdout and stderr.

(Run and return 3 values)

‘thing’ may be a string (e.g., “ls -l”) or a sequence (e.g., [“ls”, “-l”]). Internally, a string will be converted into a sequence before it is used. Any non-string items in a ‘thing’ sequence will be converted to strings using ‘str()’ (e.g., if a Label instance is given).

If ‘env’ is given, then it is the environment to use when running ‘thing’, otherwise ‘os.environ’ is used.

If ‘show_command’ is true, then “> <thing>” will be printed out before running the command.

If ‘show_output’ is true, then the output of the command (both stdout and stderr) will be printed out as the command runs.

The output of the command is shown whilst the command is running; its stdout goes to the normal stdout, and its stderr to stderr.

The command return code, stdout and stderr are returned as a tuple:

(retcode, stdout, stderr)

muddled.utils.shell(thing, env=None, show_command=True)

Run the command ‘thing’ in the shell.

If ‘thing’ is a string (e.g., “ls -l”), then it will be used as it is given.

If ‘thing’ is a sequence (e.g., [“ls”, “-l”]), then each component will be escaped with pipes.quote(), and the result concatenated (with spaces between) to give the command line to run.

If ‘env’ is given, then it is the environment to use when running ‘thing’, otherwise ‘os.environ’ is used.

If ‘show_command’ is true, then “> <thing>” will be printed out before running the command.

The output of the command will always be printed out as it runs.

If the command returns a non-zero return code, then a ShellError will be raised, containing the returncode, the command string and any output that occurred.

Unlike the various ‘runX’ functions, this calls subprocess.Popen with ‘shell=True’. This makes things like “cd” available in ‘thing’, and use of shell specific things like value expansion. It also, morei mportantly for muddle, allows commands like “git clone” to do their progress report “rolling” output. However, the warnings in the Python subprocess documentation should be heeded about not using unsafe command lines.

NB: If you do want to do “cd xxx; yyy”, you’re probably better doing:

with Directory("xxx"):
    shell("yyy")

muddled.utils.sort_domains(domains)

Given a sequence of domain names, return them sorted by depth.

So, given some random domain names (and we forgot to forbid strange names starting with ‘+’ or ‘-‘):

>>> a = ['a', '+1', '-2', 'a(b(c2))', 'a(b(c1))', '+1(+2(+4(+4)))',
...    'b(b)', 'b', 'b(a)', 'a(a)', '+1(+2)', '+1(+2(+4))', '+1(+3)']

sorting “alphabetically” gives the wrong result:

>>> sorted(a)
['+1', '+1(+2(+4(+4)))', '+1(+2(+4))', '+1(+2)', '+1(+3)', '-2', 'a', 'a(a)', 'a(b(c1))', 'a(b(c2))', 'b', 'b(a)', 'b(b)']

so we needed this function:

>>> sort_domains(a)
['+1', '+1(+2)', '+1(+2(+4))', '+1(+2(+4(+4)))', '+1(+3)', '-2', 'a', 'a(a)', 'a(b(c1))', 'a(b(c2))', 'b', 'b(a)', 'b(b)']

If we’re given a domain name that is None, we’ll replace it with ‘’.

muddled.utils.split_debian_version(v)

Takes a debian-style version string - <major>.<minor>.<subminor>-<issue><additional> - and turns it into a dictionary with those keys.

muddled.utils.split_domain(domain_name)

Given a domain name, return a tuple of the hierarchy of sub-domains.

For instance:

>>> split_domain('a')
['a']
>>> split_domain('a(b)')
['a', 'b']
>>> split_domain('a(b(c))')
['a', 'b', 'c']
>>> split_domain('a(b(c)')
Traceback (most recent call last):
...
GiveUp: Domain name "a(b(c)" has mis-matched parentheses

We don’t actually allow “sibling” sub-domains, so we try to complain helpfully:

>>> split_domain('a(b(c)(d))')
Traceback (most recent call last):
...
GiveUp: Domain name "a(b(c)(d))" has 'sibling' sub-domains

If we’re given ‘’ or None, we return [‘’], “normalising” the domain name.

>>> split_domain('')
['']
>>> split_domain(None)
['']

muddled.utils.split_path_left(in_path)

Given a path a/b/c ..., return a pair (a, b/c..) - ie. like os.path.split(), but leftward.

What we actually do here is to split the path until we have nothing left, then take the head and rest of the resulting list.

For instance:

>>> split_path_left('a/b/c')
('a', 'b/c')
>>> split_path_left('a/b')
('a', 'b')

For a single element, behave in sympathy (but, of course, reversed) to os.path.split:

>>> import os
>>> os.path.split('a')
('', 'a')
>>> split_path_left('a')
('a', '')

The empty string isn’t really a sensible input, but we cope:

>>> split_path_left('')
('', '')

And we take some care with delimiters (hopefully the right sort of care):

>>> split_path_left('/a///b/c')
('', 'a/b/c')
>>> split_path_left('//a/b/c')
('', 'a/b/c')
>>> split_path_left('///a/b/c')
('', 'a/b/c')

muddled.utils.split_vcs_url(url)

Split a URL into a vcs and a repository URL. If there’s no VCS specifier, return (None, None).

muddled.utils.string_cmp(a, b)

Return -1 if a < b, 0 if a == b, +1 if a > b.

muddled.utils.text_in_node(in_xml_node)

Return all the text in this node.

muddled.utils.total_ordering(cls)

Class decorator that fills-in missing ordering methods

muddled.utils.truncate(text, columns=None, less=0)

Truncate the given text to fit the terminal.

More specifically:

1. Split on newlines
2. If the first line is too long, cut it and add ‘...’ to the end.
3. Return the first line

If ‘columns’ is 0, then don’t do the truncation of the first line.

If ‘columns’ is None, then try to work out the current terminal width (using “curses”), and otherwise use 80.

If ‘less’ is specified, then the actual width used will be the calculated or given width, minus ‘less’ (so if columns=80 and less=2, then the maximum line length would be 78). Clearly this is ignored if ‘columns’ is 0.

muddled.utils.unescape_backslashes(str)

Replace every string ‘X’ with X, as if you were a shell

muddled.utils.unix_time()

Return the current UNIX time since the epoch.

muddled.utils.unquote_list(lst)

Given a list of objects, potentially enclosed in quotation marks or other shell weirdness, return a list of the actual objects.

muddled.utils.well_formed_dot_muddle_dir(dir)

Return True if this seems to be a well-formed .muddle directory

We’re not trying to be absolutely rigorous, but do want to detect (for instance) an erroneous file with that name, or an empty directory

muddled.utils.wrap(text, width=None, **kwargs)

A convenience wrapper around textwrap.wrap()

(basically because muddled users will have imported utils already).

muddled.utils.xml_elem_with_child(doc, elem_name, child_text)

Return an element ‘elem_name’ containing the text child_text in doc.

muddled.checkouts

muddled.checkouts.simple

Simple entry points so that descriptions can assert the existence of checkouts easily

muddled.checkouts.simple.absolute(builder, co_name, repo_url, rev=None, branch=None)

Check out a repository from an absolute URL.

<repo_url> must be of the form <vcs>+<url>, where <vcs> is one of the support version control systems (e.g., ‘git’, ‘svn’).

<rev> may be a revision (specified as a string). “HEAD” (or its equivalent) is assumed by default.

<branch> may be a branch. “master” (or its equivalent) is assumed by default.

The repository <repo_url>/<co_name> will be checked out into src/<co_name>.

muddled.checkouts.simple.relative(builder, co_name, repo_relative=None, rev=None, branch=None)

A simple, VCS-controlled, checkout.

<rev> may be a revision (specified as a string). “HEAD” (or its equivalent) is assumed by default.

<branch> may be a branch. “master” (or its equivalent) is assumed by default.

If <repo_relative> is None then the repository <base_url>/<co_name> will be checked out into src/<co_name>, where <base_url> is the base URL as specified in .muddle/RootRepository (i.e., the base URL of the build description, as used in “muddle init”).

For example:

<base_url>/<co_name>  -->  src/<co_name>

If <repo_relative> is not None, then the repository <base_url>/<repo_relative> will be checked out instead:

<base_url>/<repo_relative>  -->  src/<co_name>

muddled.checkouts.twolevel

Two-level checkouts. Makes it slightly easier to separate checkouts out into roles. I’ve deliberately not implemented arbitrary-level checkouts for fear of complicating the checkout tree.

muddled.checkouts.twolevel.absolute(builder, co_dir, co_name, repo_url, rev=None, branch=None)

Check out a twolevel repository from an absolute URL.

<repo_url> must be of the form <vcs>+<url>, where <vcs> is one of the support version control systems (e.g., ‘git’, ‘svn’).

<rev> may be a revision (specified as a string). “HEAD” (or its equivalent) is assumed by default.

<branch> may be a branch. “master” (or its equivalent) is assumed by default.

The repository <repo_url>/<co_name> will be checked out into src/<co_dir>/<co_name>.

muddled.checkouts.twolevel.relative(builder, co_dir, co_name, repo_relative=None, rev=None, branch=None)

A two-level version of checkout.simple.relative().

It attempts to check out <co_dir>/<co_name> (but see below).

<rev> may be a revision (specified as a string). “HEAD” (or its equivalent) is assumed by default.

<branch> may be a branch. “master” (or its equivalent) is assumed by default.

If <repo_relative> is None then the repository <base_url>/<co_name> will be checked out, where <base_url> is the base URL as specified in .muddle/RootRepository (i.e., the base URL of the build description, as used in “muddle init”).

If <repo_relative> is not None, then the repository <base_url>/<repo_relative> ...

In the normal case, the location in the repository and in the checkout is assumed the same (i.e., <co_dir>/<co_name>). So, for instance, with co_dir=”A” and co_name=”B”, the repository would have:

<base_url>/A/B

which we would check out into:

src/A/B

Occasionally, though, the repository is organised differently, so for instance, one might want to checkout:

<base_url>/B

into:

src/A/B

In this latter case, one can use the ‘repo_relative’ argument, to say where the checkout is relative to the repository’s “base”. So, in the example above, we still have co_dir=”A” and co_name=”B”, but we also want to say repo_relative=B.

muddled.checkouts.twolevel.twolevel(builder, co_dir, co_name, repo_relative=None, rev=None, branch=None)

A two-level version of checkout.simple.relative().

It attempts to check out <co_dir>/<co_name> (but see below).

<rev> may be a revision (specified as a string). “HEAD” (or its equivalent) is assumed by default.

<branch> may be a branch. “master” (or its equivalent) is assumed by default.

If <repo_relative> is None then the repository <base_url>/<co_name> will be checked out, where <base_url> is the base URL as specified in .muddle/RootRepository (i.e., the base URL of the build description, as used in “muddle init”).

If <repo_relative> is not None, then the repository <base_url>/<repo_relative> ...

In the normal case, the location in the repository and in the checkout is assumed the same (i.e., <co_dir>/<co_name>). So, for instance, with co_dir=”A” and co_name=”B”, the repository would have:

<base_url>/A/B

which we would check out into:

src/A/B

Occasionally, though, the repository is organised differently, so for instance, one might want to checkout:

<base_url>/B

into:

src/A/B

In this latter case, one can use the ‘repo_relative’ argument, to say where the checkout is relative to the repository’s “base”. So, in the example above, we still have co_dir=”A” and co_name=”B”, but we also want to say repo_relative=B.

muddled.deployments

File deployment indicates where the final deployment should end up. For instance:

muddled.deployments.filedep.deploy(builder, '/', 'omap', roles)

muddled.deployments.collect

Collect deployment.

This deployment is used to collect elements from:

• checkout directories
• package ‘obj’ directories
• package role ‘install’ directories
• other deployments

into deployment directories, usually to be processed by some external tool.

class muddled.deployments.collect.AssemblyDescriptor(from_label, from_rel, to_name, recursive=True, failOnAbsentSource=False, copyExactly=True, usingRSync=False, obeyInstructions=True)

Bases: object

Construct an assembly descriptor.

We copy from the directory from_rel in from_label (package, deployment, checkout) to the name to_name under the deployment.

Give a package of ‘*’ to copy from the install directory for a given role.

If recursive is True, we’ll copy recursively.

• failOnAbsentSource - If True, we’ll fail if the source doesn’t exist.
• copyExactly - If True, keeps links. If false, copies the file they point to.

get_source_dir(builder)

class muddled.deployments.collect.CollectApplyChmod

Bases: muddled.deployments.collect.InstructionImplementor

apply(builder, instr, role, path)

needs_privilege(builder, instr, role, path)

prepare(builder, instr, role, path)

class muddled.deployments.collect.CollectApplyChown

Bases: muddled.deployments.collect.InstructionImplementor

apply(builder, instr, role, path)

needs_privilege(builder, instr, role, path)

prepare(builder, instr, role, path)

class muddled.deployments.collect.CollectDeploymentBuilder

Bases: muddled.depend.Action

Builds the specified collect deployment.

add_assembly(assembly_descriptor)

apply_instructions(builder, label, prepare, deploy_path)

build_label(builder, label)

Actually do the copies ..

deploy(builder, label, target_base)

sort_out_and_run_instructions(builder, label)

class muddled.deployments.collect.InstructionImplementor

Bases: object

apply(builder, instruction, role, path)

needs_privilege(builder, instr, role, path)

prepare(builder, instruction, role, path)

Prepares for rsync. This means fixing up the destination file (e.g. removing it if it may have changed uid by a previous deploy) so we will be able to rsync it.

muddled.deployments.collect.copy_from_checkout(builder, name, checkout, rel, dest, recursive=True, failOnAbsentSource=False, copyExactly=True, domain=None, usingRSync=False)

muddled.deployments.collect.copy_from_deployment(builder, name, dep_name, rel, dest, recursive=True, failOnAbsentSource=False, copyExactly=True, domain=None, usingRSync=False)

usingRSync - set to True to copy with rsync - substantially faster than

cp

muddled.deployments.collect.copy_from_package_obj(builder, name, pkg_name, pkg_role, rel, dest, recursive=True, failOnAbsentSource=False, copyExactly=True, domain=None, usingRSync=False)

• If ‘usingRSync’ is true, copy with rsync - substantially faster than

  cp, if you have rsync. Not very functional if you don’t :-)

muddled.deployments.collect.copy_from_role_install(builder, name, role, rel, dest, recursive=True, failOnAbsentSource=False, copyExactly=True, domain=None, usingRSync=False, obeyInstructions=True)

Add a requirement to copy from the given role’s install to the named deployment.

‘name’ is the name of the collecting deployment, as created by:

deploy(builder, name)

which is remembered as a rule whose target is deployment:<name>/deployed, where <name> is the ‘name’ given.

‘role’ is the role to copy from. Copying will be based from ‘rel’ within the role’s install, to ‘dest’ within the deployment.

The label package:(<domain>)*{<role>}/postinstalled will be added as a dependency of the collecting deployment rule.

An AssemblyDescriptor will be created to copy from ‘rel’ in the install directory of the label package:*{<role>}/postinstalled, to ‘dest’ within the deployment directory of ‘name’, and added to the rule’s actions.

So, for instance:

copy_from_role_install(builder,'fred','data','public','data/public',
                       True, False, True)

might copy (recursively) from:

install/data/public

to:

deploy/fred/data/public

‘rel’ may be the empty string (‘’) to copy all files in the install directory.

• If ‘recursive’ is true, then copying is recursive, otherwise it is not.
• If ‘failOnAbsentSource’ is true, then copying will fail if the source does not exist.
• If ‘copyExactly’ is true, then symbolic links will be copied as such, otherwise the linked file will be copied.

• If ‘usingRSync’ is true, copy with rsync - substantially faster than

  cp, if you have rsync. Not very functional if you don’t :-)

• If ‘obeyInstructions’ is False, don’t obey any applicable instructions.

muddled.deployments.collect.deploy(builder, name)

Create a collection deployment builder.

This adds a new rule linking the label deployment:<name>/deployed to the collection deployment builder.

You can then add assembly descriptors using the other utility functions in this module.

Dependencies get registered when you add an assembly descriptor.

muddled.deployments.cpio

cpio deployment.

Most commonly used to create Linux ramdisks, this deployment creates a CPIO archive from the relevant install directory and applies the relevant instructions.

Because python has no native CPIO support, we need to do this by creating a tar archive and then invoking cpio in copy-through mode to convert the archive to cpio. Ugh.

class muddled.deployments.cpio.CIApplyChmod

Bases: muddled.deployments.cpio.CpioInstructionImplementor

apply(builder, instr, role, target_base, hierarchy)

class muddled.deployments.cpio.CIApplyChown

Bases: muddled.deployments.cpio.CpioInstructionImplementor

apply(builder, instr, role, target_base, hierarchy)

class muddled.deployments.cpio.CIApplyMknod

Bases: muddled.deployments.cpio.CpioInstructionImplementor

apply(builder, instr, role, target_base, hierarchy)

class muddled.deployments.cpio.CpioDeploymentBuilder(target_file, target_base, compressionMethod=None, pruneFunc=None)

Bases: muddled.depend.Action

Builds the specified CPIO deployment.

• ‘target_file’ is the CPIO file to construct.
• ‘target_base’ is an array of pairs mapping labels to target locations, or (label, src) -> location
• ‘compressionMethod’ is the compression method to use, if any - gzip -> gzip, bzip2 -> bzip2.
• if ‘pruneFunc’ is not None, it is a function to be called like pruneFunc(Hierarchy) to prune the hierarchy prior to packing. Usually something like deb.deb_prune, it’s intended to remove spurious stuff like manpages from initrds and the like.

attach_env(builder)

Attaches an environment containing:

MUDDLE_TARGET_LOCATION - the location in the target filesystem where this deployment will end up.

to every package label in this role.

build_label(builder, label)

Actually cpio everything up, following instructions appropriately.

class muddled.deployments.cpio.CpioInstructionImplementor

Bases: object

apply(builder, instruction, role, path)

class muddled.deployments.cpio.CpioWrapper(builder, action, label)

Bases: object

copy_from_role(from_role, from_fragment, to_fragment, with_base=None)

Copy the relative path from_fragment in from_role to to_fragment in the CPIO package or deployment given by ‘action’

Use ‘with_base’ to change the base offset we apply when executing instructions; this is useful when using repeated copy_from_role() invocations to copy a subset of one role to a package/deployment.

done()

Call this once you’ve added all the roles you want; it attaches the deployment environment to them and generally finishes up

muddled.deployments.cpio.create(builder, target_file, name, compressionMethod=None, pruneFunc=None)

Create a CPIO deployment and return it.

• ‘builder’ is the muddle builder that is driving us

• ‘target_file’ is the name of the CPIO file we want to create. Note that this may include a sub-path (for instance, “fred/file.cpio” or even “/fred/file.cpio”).

• ‘name’ is either:

  1. The name of the deployment that will contain this CPIO file (in the builder’s default domain), or
  2. A deployment or package label, ditto

• ‘comporessionMethod’ is the compression method to use:

  • None means no compression
  • ‘gzip’ means gzip
  • ‘bzip2’ means bzip2

• if ‘pruneFunc’ is not None, it is a function to be called like pruneFunc(Hierarchy) to prune the hierarchy prior to packing. Usually something like deb.deb_prune, it’s intended to remove spurious stuff like manpages from initrds and the like.

Normal usage is thus something like:

fw = cpio.create(builder, 'firmware.cpio', deployment)
fw.copy_from_role(role1, '', '/')
fw.copy_from_role(role2, 'bin', '/bin')
fw.done()

or:

fw = cpio.create(builder, 'firmware.cpio', package('firmware', role))
fw.copy_from_role(role, '', '/')
fw.done()

muddled.deployments.filedep

File deployment. This deployment just copies files into a role subdirectory in the /deployed directory, applying appropriate instructions.

class muddled.deployments.filedep.FIApplyChmod

Bases: muddled.deployments.collect.CollectApplyChmod

needs_privilege(builder, instr, role, path)

class muddled.deployments.filedep.FIApplyMknod

Bases: muddled.deployments.collect.InstructionImplementor

apply(builder, instr, role, path)

needs_privilege(builder, instr, role, path)

prepare(builder, instr, role, path)

class muddled.deployments.filedep.FileDeploymentBuilder(roles, target_dir)

Bases: muddled.depend.Action

Builds the specified file deployment

role is actually a list of (role, domain) pairs.

apply_instructions(builder, label)

attach_env(builder)

Attaches an environment containing:

MUDDLE_TARGET_LOCATION - the location in the target filesystem where this deployment will end up.

to every package label in this role.

build_label(builder, label)

Performs the actual build.

We actually do need to copy all files from install/ (where unprivileged processes can modify them) to deploy/ (where they can’t).

Then we apply instructions to deploy.

deploy(builder, label)

muddled.deployments.filedep.deploy(builder, target_dir, name, roles)

Register a file deployment.

This is a convenience wrapper around deploy_with_domains().

‘roles’ is a sequence of role names. The deployment will take the roles specified, and build them into a deployment at deploy/[name].

More specifically, a rule will be created for label:

“deployment:<name>/deployed”

which depends on “package:*{<role}/postinstalled” (in the builder’s default domain) for each <role> in ‘roles’.

In other words, the deployment called ‘name’ will depend on the given roles (in the default domain) having been “finished” (postinstalled).

An “instructions applied” label “deployment:<name>/instructionsapplied” will also be created.

The deployment should eventually be located at ‘target_dir’.

muddled.deployments.filedep.deploy_with_domains(builder, target_dir, name, role_domains)

Register a file deployment.

‘role_domains’ is a sequence of (role, domain) pairs. The deployment will take the roles and domains specified, and build them into a deployment at deploy/[name].

More specifically, a rule will be created for label:

“deployment:<name>/deployed”

which depends on “package:(<domain>)*{<role}/postinstalled” for each (<role>, <domain>) pair in ‘role_domains’.

In other words, the deployment called ‘name’ will depend on the given roles in the appropriate domains having been “finished” (postinstalled).

An “instructions applied” label “deployment:<name>/instructionsapplied” will also be created.

The deployment should eventually be located at ‘target_dir’.

muddled.deployments.tools

Tools deployment. This deployment merely adds the appropriate environment variables to use the tools in the given role install directories to everything in another list of deployments.

Instructions are ignored - there’s no reason to follow them (yet) and it’s simpler not to.

XXX Do we support this anymore?

class muddled.deployments.tools.ToolsDeploymentBuilder(dependent_roles)

Bases: muddled.depend.Action

Copy the dependent roles into the tools deployment.

build_label(builder, label)

deploy(builder, label)

muddled.deployments.tools.attach_env(builder, role, env, name)

Attach suitable environment variables for the given input role to the given environment store.

We set:

• LD_LIBRARY_PATH - Prepend $role_installl/lib
• PATH - Append $role_install/bin
• PKG_CONFIG_PATH - Prepend $role_install/lib/pkgconfig
• $role_TOOLS_PATH - Prepend $role_install/bin

The PATH/TOOLS_PATH stuff is so you can still locate tools which were in the path even if they’ve been overridden with your built tools.

muddled.deployments.tools.deploy(builder, name, rolesThatUseThis=[], rolesNeededForThis=[])

Register a tools deployment.

This is used to:

1. Set the environment for each role in ‘rolesThatUseThis’ so that PATH, LD_LIBRARY_PATH and PKG_CONFIG_PATH include the ‘name’ deployment
2. Make deployment:<name>/deployed depend upon the ‘rolesNeededForThis’
3. Register cleanup for this deployment

The intent is that we have a “tools” deployment, which provides useful host tools (for instance, something to mangle a file in a particular manner). Those roles which need to use such tools in their builds (normally in a Makefile.muddle) then need to have the environment set appropriately to allow them to find the tools (and ideally, not system provided tools which mighth have the same name).

muddled.pkgs

muddled.pkgs.aptget

An apt-get package. When you try to build it, this package pulls in a pre-canned set of packages via apt-get.

class muddled.pkgs.aptget.AptGetBuilder(name, role, pkgs_to_install, os_version=None)

Bases: muddled.pkg.PackageBuilder

Make sure that particular OS packages have been installed.

The “build” action for AptGetBuilder uses the Debian tool apt-get to ensure that each package is installed.

Our arguments are:

• ‘name’ - the name of this builder
• ‘role’ - the role to which it belongs
• ‘pkgs_to_install’ - a sequence specifying which packages are to be installed.

Each item in the sequence ‘pkgs_to_install’ can be:

• the name of an OS package to install - for instance, ‘libxml2-dev’

  (this is backwards compatible with how this class worked in the past)

• a Choice allowing a particular package to be selected according to the operating system.

  See “muddle doc Choice” for details on the Choice class.

  Note that a choice resulting in None (i.e., where the default value is None, and the default is selected) will not do anything.

  If ‘os_version’ is given, then it will be used as the version name, otherwise the result of calling utils.get_os_version_name() will be used.

We also allow a single string, or a single Choice, treated as if they were wrapped in a list.

already_installed(pkg)

Decide if the quoted debian package is already installed.

We use dpkg-query:

$ dpkg-query -W -f=\${Status}\n libreadline-dev
install ok installed

That third word means what it says (installed). Contrast with a package that is either not recognised or has not been downloaded at all:

$ dpkg-query -W -f=\${Status}\n a0d
dpkg-query: no packages found matching a0d

So we do some fairly simple processing of the output...

build_label(builder, label)

This time, build is the only one we care about.

muddled.pkgs.aptget.depends_on_aptget(builder, name, role, pkg, pkg_role)

Make a package dependant on a particular apt-builder.

• pkg - The package we want to add a dependency to. ‘*’ is a good thing to add here ..

muddled.pkgs.aptget.medium(builder, name, role, apt_pkgs, roles, os_version=None)

Construct an apt-get package and make every package in the named roles depend on it.

Note that apt_pkgs can be an OS package name or a choices sequence - see the documentation for AptGetBuilder.

muddled.pkgs.aptget.simple(builder, name, role, apt_pkgs, os_version=None)

Construct an apt-get package in the given role with the given apt_pkgs.

Note that apt_pkgs can be an OS package name or a Choice - see the documentation for AptGetBuilder for more details.

For instance (note: not a real example - the dependencies don’t make sense!):

from muddled.utils import Choice
from muddled.pkgs import aptget
aptget.simple(builder, "host_packages", "host_environment",
       [
       "gcc-multilib",
       "g++-multilib",
       "lib32ncurses5-dev",
       "lib32z1-dev",
       "bison",
       "flex",
       "gperf",
       "libx11-dev",
       # On Ubuntu 11 or 12, choose icedtea-7, otherwise icedtea-6
       Choice([ ("ubuntu 1[12].*", "icedtea-7-jre"),
                ("ubuntu *", "icedtea-6-jre") ]),
       # On Ubuntu 10 or later, use libgtiff5
       # On Ubuntu 3 through 9, use libgtiff4
       # Otherwise, just don't try to use libgtiff
       Choice([ ("ubuntu 1?", "libgtiff5"),
                ("ubuntu [3456789]", "libgtiff4"),
                None ])
       ])

muddled.pkgs.deb

Some code which sneakily steals binaries from Debian/Ubuntu.

Quite a lot of code for embedded systems can be grabbed pretty much directly from the relevant Ubuntu binary packages - this won’t work with complex packages like exim4 without some external frobulation, since they have relatively complex postinstall steps, but it works supported architecture it’s a quick route to externally maintained binaries which actually work and it avoids having to build absolutely everything in your linux yourself.

This package allows you to ‘build’ a package from a source file in a checkout which is a .deb. We run dpkg with enough force options to install it in the relevant install directory.

You still need to provide any relevant instruction files (we’ll register <filename>.instructions.xml for you automatically if it exists).

We basically ignore the package database (there is one, but it’s always empty and stored in the object directory).

class muddled.pkgs.deb.DebAction(name, role, co, pkg_name, pkg_file, instr_name=None, postInstallMakefile=None)

Bases: muddled.pkg.PackageBuilder

Use dpkg to extract debian archives from the given checkout into the install directory.

• co - is the checkout name in which the package resides.

• pkg_name - is the name of the package (dpkg needs it)

• pkg_file - is the name of the file the package is in, relative to the checkout directory.

• instr_name - is the name of the instruction file, if any.

• postInstallMakefile - if not None:

  make -f postInstallMakefile <pkg-name>
  

  will be run at post-install time to make links, etc.

build_label(builder, label)

Build the relevant label.

ensure_dirs(builder, label)

class muddled.pkgs.deb.DebDevAction(name, role, co, pkg_name, pkg_file, instr_name=None, postInstallMakefile=None, nonDevCoName=None, nonDevPkgFile=None)

Bases: muddled.pkg.PackageBuilder

Use dpkg to extract debian archives into obj/include and obj/lib directories so we can use them to build other packages.

As for a DebAction, really.

build_label(builder, label)

Actually install the dev package.

ensure_dirs(builder, label)

muddled.pkgs.deb.deb_prune(h)

Given a cpiofile hierarchy, prune it so that only the useful stuff is left.

We do this by lopping off directories, which is easy enough in cpiofile heirarchies.

muddled.pkgs.deb.dev(builder, coName, name, roles, depends_on=[], pkgFile=None, debName=None, nonDevCoName=None, nonDevDebName=None, instrFile=None, postInstallMakefile=None)

A wrapper for ‘deb.simple’, with the “idDev” flag set True.

• nonDevCoName is the checkout in which the non-dev version of the package resides.
• nonDevDebName is the non-dev version of the package; this is sometimes needed because of the odd way in which debian packages the ‘.so’ link in the dev package and the sofiles themselves into the non-dev.

muddled.pkgs.deb.extract_into_obj(inv, co_name, label, pkg_file)

muddled.pkgs.deb.rewrite_links(inv, label)

muddled.pkgs.deb.simple(builder, coName, name, roles, depends_on=[], pkgFile=None, debName=None, instrFile=None, postInstallMakefile=None, isDev=False, nonDevCoName=None, nonDevPkgFile=None)

Build a package called ‘name’ from co_name / pkg_file with an instruction file called instr_file.

‘name’ is the name of the muddle package and of the debian package. if you want them different, set deb_name to something other than None.

Set isDev to True for a dev package, False for an ordinary binary package. Dev packages are installed into the object directory where MUDDLE_INC_DIRS etc. expects to look for them. Actual packages are installed into the installation directory where they will be transported to the target system.

muddled.pkgs.initscripts

Write an initialisation script into $(MUDDLE_TARGET_LOCATION)/bin/$(something)

This is really just using utils.subst_file() with the current environment, on a resource stored in resources/.

We also write a setvars script with a suitable set of variables for running code in the context of the deployment, and any variables you’ve set in the environment store retrieved with get_env_store().

class muddled.pkgs.initscripts.InitScriptBuilder(name, role, script_name, deployments, writeSetvarsSh=True, writeSetvarsPy=False)

Bases: muddled.pkg.PackageBuilder

Build an init script.

build_label(builder, label)

Install is the only one we care about ..

muddled.pkgs.initscripts.get_effective_env(builder, name, role, domain=None)

Retrieve the effective runtime environment for this initscripts package. Note that setting variables here will have no effect.

muddled.pkgs.initscripts.get_env(builder, name, role, domain=None)

Retrieve an environment to which you can make changes which will be reflected in the generated init scripts. The actual environment used will have extra values inserted from wildcarded environments - see get_effective_env() above.

muddled.pkgs.initscripts.medium(builder, name, roles, script_name, deployments=[], writeSetvarsSh=True, writeSetvarsPy=False)

Build an init script for the given roles.

muddled.pkgs.initscripts.setup_default_env(builder, env_store)

Set up the default environment for this initscript.

muddled.pkgs.initscripts.simple(builder, name, role, script_name, deployments=[], writeSetvarsSh=True, writeSetvarsPy=False)

Build an init script for the given role.

muddled.pkgs.make

Some standard package implementations to cope with packages that use Make

class muddled.pkgs.make.ExpandingMakeBuilder(name, role, co_name, archive_file, archive_dir, makefile='Makefile.muddle')

Bases: muddled.pkgs.make.MakeBuilder

A MakeBuilder that first expands an archive file.

A MakeBuilder that first expands an archive file.

For package ‘name’ in role ‘role’, look in checkout ‘co_name’ for archive ‘archive_file’. Unpack that into $MUDDLE_OBJ, as ‘archive_dir’, with ‘obj/’ linked to it, and use ‘makefile’ to build it.

build_label(builder, label)

Build our label.

Cleverly, Richard didn’t define anything for MakeBuilder to do at the PreConfigure step, which means we can safely do whatever we need to do in this subclass...

unpack_archive(builder, label)

class muddled.pkgs.make.MakeBuilder(name, role, co, config=True, perRoleMakefiles=False, makefileName='Makefile.muddle', rewriteAutoconf=False, usesAutoconf=False, execRelPath=None)

Bases: muddled.pkg.PackageBuilder

Use make to build your package from the given checkout.

We assume that the makefile is smart enough to build in the object directory, since any other strategy (e.g. convolutions involving cp) will lead to dependency-based disaster.

Constructor for the make package.

build_label(builder, label)

Build the relevant label. We’ll assume that the checkout actually exists.

ensure_dirs(builder, label)

Make sure all the relevant directories exist.

muddled.pkgs.make.attach_env(builder, name, role, checkout, domain=None)

Write the environment which attaches MUDDLE_SRC to makefiles.

We retrieve the environment for package:<name>{<role>}/*, and set MUDDLE_SRC therein to the checkout path for ‘checkout:<checkout>’.

muddled.pkgs.make.deduce_makefile_name(makefile_name, per_role, role)

Deduce our actual muddle Makefile name.

‘makefile_name’ is the base name. If it is None, then we use DEFAULT_MAKEFILE_NAME.

If ‘per_role’ is true, and ‘role’ is not None, then we add the extension ‘.<role>’ to the end of the makefile name.

Abstracted here so that it can be used outside this module as well.

muddled.pkgs.make.expanding_package(builder, name, archive_dir, role, co_name, co_dir, makefile='Makefile.muddle', deps=None, archive_file=None, archive_ext='.tar.bz2')

Specify how to expand and build an archive file.

As normal, ‘name’ is the package name, ‘role’ is the role to build it in, ‘co_name’ is the name of the checkout, and ‘co_dir’ is the directory in which that lives.

We expect to unpack an archive

<co_dir>/<co_name>/<archive_dir><archive_ext>

into $(MUDDLE_OBJ)/<archive_dir>. (NB: if the archive file does not expand into a directory of the obvious name, you can specify the archive file name separately, using ‘archive_file’).

So, for instance, all of our X11 “stuff” lives in checkout “X11R7.5” which is put into directory “x11” – i.e., “src/X11/X11R7.5”.

That lets us keep stuff together in the repository, without leading to a great many packages that are of no direct interest to anyone else.

Within that we then have various muddle makefiles, and a set of .tar.bz archive files.

1. The archive file expands into a directory called ‘archive_dir’
2. It is assumed that the archive file is named ‘archive_dir’ + ‘archive_ext’. If this is not so, then specify ‘archive_file’ (and/or ‘archive_ext’) appropriately.

This function is used to say: take the named archive file, use package name ‘name’, unpack the archive file into $(MUDDLE_OBJ_OBJ), and build it using the named muddle makefile.

This allows various things to be build with the same makefile, which is useful for (for instance) X11 proto[type] archives.

Note that in $(MUDDLE_OBJ), ‘obj’ (i.e., $(MUDDLE_OBJ_OBJ)) will be a soft link to the expanded archive directory.

muddled.pkgs.make.medium(builder, name, roles, checkout, rev=None, branch=None, deps=None, dep_tag='preconfig', simpleCheckout=True, config=True, perRoleMakefiles=False, makefileName='Makefile.muddle', usesAutoconf=False, rewriteAutoconf=False, execRelPath=None)

Build a package controlled by make, in the given roles with the given dependencies in each role.

• simpleCheckout - If True, register the checkout as simple checkout too.
• dep_tag - The tag to depend on being installed before you’ll build.
• perRoleMakefiles - If True, we run ‘make -f Makefile.<rolename>’ instead of just make.

muddled.pkgs.make.multilevel(builder, name, roles, co_dir=None, co_name=None, rev=None, branch=None, deps=None, dep_tag='preconfig', simpleCheckout=True, config=True, perRoleMakefiles=False, makefileName='Makefile.muddle', repo_relative=None, usesAutoconf=False, rewriteAutoconf=False, execRelPath=None)

Build a package controlled by make, in the given roles with the given dependencies in each role.

• simpleCheckout - If True, register the checkout as simple checkout too.
• dep_tag - The tag to depend on being installed before you’ll build.
• perRoleMakefiles - If True, we run ‘make -f Makefile.<rolename>’ instead of just make.

muddled.pkgs.make.simple(builder, name, role, checkout, rev=None, branch=None, simpleCheckout=False, config=True, perRoleMakefiles=False, makefileName='Makefile.muddle', usesAutoconf=False, rewriteAutoconf=False, execRelPath=None)

Build a package controlled by make, called name with role role from the sources in checkout checkout.

• simpleCheckout - If True, register the checkout too.
• config - If True, we have make config. If false, we don’t.
• perRoleMakefiles - If True, we run ‘make -f Makefile.<rolename>’ instead of just make.

• usesAutoconf - If True, this package is given access to .la and .pc

  files from things it depends on.

• rewriteAutoconf - If True, we will rewrite .la and .pc files in the output directory so that packages which use autoconf continue to depend correctly. Intended for use with the MUDDLE_PKGCONFIG_DIRS environment variable.

• execRelPath - Where, relative to the object directory, do we find

  binaries for this package?

muddled.pkgs.make.single(builder, name, role, deps=None, usesAutoconf=False, rewriteAutoconf=False, execRelPath=None)

A simple make package with a single checkout named after the package and a single role.

muddled.pkgs.make.twolevel(builder, name, roles, co_dir=None, co_name=None, rev=None, branch=None, deps=None, dep_tag='preconfig', simpleCheckout=True, config=True, perRoleMakefiles=False, makefileName='Makefile.muddle', repo_relative=None, usesAutoconf=False, rewriteAutoconf=False, execRelPath=None)

Build a package controlled by make, in the given roles with the given dependencies in each role.

• simpleCheckout - If True, register the checkout as simple checkout too.
• dep_tag - The tag to depend on being installed before you’ll build.
• perRoleMakefiles - If True, we run ‘make -f Makefile.<rolename>’ instead of just make.