#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# slackroll - A package or upgrade manager for Slackware Linux.
#
# Written in 2007,2009-2014,2017 by Ricardo Garcia <r@rg3.name>.
# Contributions in 2017-2018,2021 by Andrew Clemons <andrew.clemons@gmail.com>.
#
# To the extent possible under law, the author(s) have dedicated all copyright
# and related and neighboring rights to this software to the public domain
# worldwide. This software is distributed without any warranty.
#
# You should have received a copy of the CC0 Public Domain Dedication along with
# this software. If not, see
# <http://creativecommons.org/publicdomain/zero/1.0/>.
#
import anydbm
import bz2
import cPickle
import cStringIO
import fcntl
import functools
import glob
import math
import operator
import os
import os.path
import pwd
import re
import shelve
import socket
import struct
import subprocess
import sys
import termios
import urllib
import urlparse

slackroll_version = 49

slackroll_exit_failure = 1
slackroll_exit_success = 0

slackroll_pkg_re = re.compile(r'^(.*/)?([^/]+)-([^/-]+)-([^/-]+)-([^/-]+?)(\.t[bglx]z)?$')
slackroll_pkg_re_path = 1
slackroll_pkg_re_name = 2
slackroll_pkg_re_version = 3
slackroll_pkg_re_arch = 4
slackroll_pkg_re_build = 5
slackroll_pkg_re_suffix = 6

slackroll_base_dir = os.path.join(os.path.sep, 'var', 'slackroll')
slackroll_default_temp_dir = os.path.join(slackroll_base_dir, 'tmp')
slackroll_pkgs_dir_glob = os.path.join(slackroll_base_dir, 'packages', '*')
slackroll_pkgs_dir = os.path.dirname(slackroll_pkgs_dir_glob)
slackroll_help_file_template = os.path.join(os.path.sep, 'usr', 'doc', 'slackroll-v%s' % slackroll_version, 'helpfiles', '%s.txt')
slackroll_local_pkgs_dir_names = [os.path.join(os.path.sep, 'var', 'lib', 'pkgtools', 'packages'), os.path.join(os.path.sep, 'var', 'log', 'packages')]
slackroll_local_pkg_filelist_marker = 'FILE LIST:\n'
slackroll_filelist_pkg_re = re.compile(
        # Output line in ls form
        r'^-[rwx-]{9}\s+\d+\s+\w+\s+\w+\s+(\d+)\s+'                 # Everything up to the date
        r'\S+(?:\s+\S+){1,2}'                                       # Date itself in several forms (two or three "words")
        r'\s+(\./(?:.+/)?[^/]+-[^/-]+-[^/-]+-[^/-]+\.t[bglx]z)$'    # Package path
)
slackroll_filelist_pkg_size = 1
slackroll_filelist_pkg_str = 2
slackroll_source_indicator = '/source/'
slackroll_pasture_indicator = '/pasture/'
slackroll_patch_indicator = '/patches/'
slackroll_main_indicator = re.compile(r'/slackware(?:64|arm)?/')
slackroll_extra_indicator = '/extra/'
slackroll_prioritized_pkgs = ['aaa_glibc-solibs', 'glibc-solibs', 'sed', 'pkgtools']
slackroll_kernel_pkg_indicator = 'kernel'

slackroll_self_filename = os.path.join(slackroll_base_dir, 'self')
slackroll_mirror_filename = os.path.join(slackroll_base_dir, 'mirror')
slackroll_primary_mirror_filename = os.path.join(slackroll_base_dir, 'pmirror')
slackroll_persistentlist_filename = os.path.join(slackroll_base_dir, 'persistent.db')
slackroll_locallist_filename = os.path.join(slackroll_base_dir, 'local.db')
slackroll_remotelist_filename = os.path.join(slackroll_base_dir, 'remote.db')
slackroll_pkg_files_filename = os.path.join(slackroll_base_dir, 'pkgfiles.db')
slackroll_known_files = os.path.join(slackroll_base_dir, 'knownfiles.db')
slackroll_blacklist_filename = os.path.join(slackroll_base_dir, 'blacklist.db')
slackroll_repolist_filename = os.path.join(slackroll_base_dir, 'repos.db')
slackroll_filelist_filename = 'FILELIST.TXT'
slackroll_changelog_filename = 'ChangeLog.txt'
slackroll_changelog_entry_separator = '+--------------------------+\n'
slackroll_local_filelist = os.path.join(slackroll_base_dir, slackroll_filelist_filename)
slackroll_local_changelog = os.path.join(slackroll_base_dir, 'changelog.db')
slackroll_gpgkey_filename = 'GPG-KEY'
slackroll_local_gpgkey = os.path.join(slackroll_base_dir, slackroll_gpgkey_filename)
slackroll_temp_suffix = '.part'
slackroll_signature_suffix = '.asc'
slackroll_info_suffix = '.txt'
slackroll_new_suffix = '.new'
slackroll_etc_dir = '/etc'
slackroll_pkg_archive_suffix = re.compile(r'\.t[bglx]z$')
slackroll_default_pager = 'less'
slackroll_default_visual = 'vim'
slackroll_default_difftool = 'vimdiff'
slackroll_gnupg_exec_names = ['gpg2', 'gpg']
slackroll_never_missing_re = re.compile(r'(^/install/)|(^/dev/)|(^/lib/incoming/)')

slackroll_state_new = 0
slackroll_state_unavailable = 1
slackroll_state_installed = 2
slackroll_state_notinstalled = 3
slackroll_state_frozen = 4
slackroll_state_foreign = 5
slackroll_state_outdated = 6
slackroll_state_strings = ['new', 'unavailable', 'installed', 'not-installed', 'frozen', 'foreign', 'outdated']
slackroll_transient_states = [slackroll_state_new, slackroll_state_outdated, slackroll_state_unavailable]
slackroll_all_states = [x for x in xrange(len(slackroll_state_strings))]

slackroll_socket_timeout = 120
slackroll_default_primary_site_url = 'http://ftp.slackware.com/pub/slackware/slackware%s-%s/'
slackroll_arm_primary_site_url = 'http://ftp.arm.slackware.com/slackwarearm/slackware%s-%s/'
slackroll_mirror_version_re = re.compile(r'/slackware(64|arm|)-([^/]+)/')

slackroll_locale_envvars = ['LANG', 'LC_CTYPE', 'LC_NUMERIC', 'LC_TIME', 'LC_COLLATE', 'LC_MONETARY', 'LC_MESSAGES', 'LC_ALL', 'LC_PAPER', 'LC_NAME', 'LC_ADDRESS', 'LC_TELEPHONE', 'LC_MEASUREMENT', 'LC_IDENTIFICATION']
slackroll_locale_mainvar = 'LANG'
slackroll_locale_value = 'C'

slackroll_bootloader_config_files = ['/etc/lilo.conf', '/boot/grub/menu.lst', '/boot/extlinux.conf', '/boot/extlinux/extlinux.conf']
slackroll_lilo_path = '/sbin/lilo'

slackroll_manifest_re = re.compile(r'\s+(\./(?:\w+/)?MANIFEST\.bz2)$')
slackroll_manifest_basename = 'MANIFEST.bz2'
slackroll_manifest_filename = os.path.join(slackroll_base_dir, 'manifest.db')
slackroll_manifest_list_filename = os.path.join(slackroll_base_dir, 'manifestlist.db')
slackroll_batch_mode = False

def yield_slackroll_local_pkgs_dir():
    for opt in slackroll_local_pkgs_dir_names:
        while os.path.isdir(opt):
            yield opt
    sys.exit('ERROR: unable to find local packages directgory')

slackroll_local_pkgs_glob = os.path.join(yield_slackroll_local_pkgs_dir().next(), '*')
slackroll_local_pkgs_dir = os.path.dirname(slackroll_local_pkgs_glob)

def standarize_locales():
    for varname in slackroll_locale_envvars:
        if varname in os.environ:
            del os.environ[varname]
    os.environ[slackroll_locale_mainvar] = slackroll_locale_value

def index_of(thelist, elem):
    try:
        return thelist.index(elem)
    except ValueError:
        return -1

def pkg_name_cmp(name1, name2): # To be used for sorting functions
    idx1 = index_of(slackroll_prioritized_pkgs, name1)
    idx2 = index_of(slackroll_prioritized_pkgs, name2)

    if idx1 == -1:
        if idx2 == -1:
            return cmp(name1, name2)
        return 1
    if idx2 == -1:
        return -1
    return (idx1 - idx2)

def transient_cmp((name1, state1), (name2, state2)): # Special sort for transient list
    if name1 in slackroll_prioritized_pkgs or name2 in slackroll_prioritized_pkgs:
        return pkg_name_cmp(name1, name2)
    if state1 != state2:
        return (index_of(slackroll_transient_states, state1) - index_of(slackroll_transient_states, state2))
    return cmp(name1, name2)

class SlackrollError(Exception):
    pass

class SlackrollBatchModeError(SlackrollError):
    pass

class SlackwarePackage(object):
    __name = None
    __version = None
    __arch = None
    __build = None
    __path = None
    __suffix = None
    __size = None
    __base_url = None

    def __init__(self, name, version, arch, build, path, suffix, size, url):
        self.__name = name
        self.__version = version
        self.__arch = arch
        self.__build = build
        self.__path = (path is not None and path or '')
        self.__suffix = (suffix is not None and suffix or '')
        self.__size = size
        self.__base_url = url

    def __eq__(self, other):
        return self.__name == other.__name and self.__version == other.__version and self.__arch == other.__arch and self.__build == other.__build

    @property
    def name(self):
        return self.__name

    @property
    def version(self):
        return self.__version

    @property
    def arch(self):
        return self.__arch

    @property
    def build(self):
        return self.__build

    @property
    def path(self):
        return self.__path

    @property
    def suffix(self):
        return self.__suffix

    @property
    def size(self):
        return self.__size

    @property
    def idname(self):
        return '%s-%s-%s-%s' % (self.__name, self.__version, self.__arch, self.__build)

    @property
    def archivename(self):
        return '%s%s' % (self.idname, self.suffix)

    @property
    def signame(self):
        return '%s%s' % (self.archivename, slackroll_signature_suffix)

    @property
    def fullname(self):
        return os.path.join(self.path, '%s-%s-%s-%s%s' % (self.__name, self.__version, self.__arch, self.__build, self.__suffix))

    @property
    def fullsigname(self):
        return '%s%s' % (self.fullname, slackroll_signature_suffix)

    def __cmp__(self, other):
        return pkg_name_cmp(self.__name, other.__name)

    def local(self): # Returns the equivalent local pkg
        return SlackwarePackage(self.__name, self.__version, self.__arch, self.__build, slackroll_local_pkgs_dir, None, None, None)

    def base_url(self, mirror):
        # An official package doesn't have a base URL, because it comes from the
        # current configured mirror and primary mirror as given by the user.
        # This could change in the future, but would require an "update"
        # everytime the user changes mirrors, so it will probably stay this way.
        #
        # An unofficial package does have a base URL (corresponding to its
        # repository) and will take precedence above the mirror passed to this
        # method. Signatures for repository packages are downloaded from the
        # repository and not the primary mirror.
        if self.__base_url is None:
            return mirror
        return self.__base_url

    def url(self, mirror):
        return urlparse.urljoin(self.base_url(mirror), self.fullname)

    def sig_url(self, mirror):
        return urlparse.urljoin(self.base_url(mirror), self.fullsigname)

class SlackrollURLopener(urllib.FancyURLopener):
    def http_error_default(self, url, fp, errcode, errmsg, headers):
        raise IOError('%s: %s' % (errcode, errmsg))

def pkg_from_str(path_or_name): # Create a SlackwarePackage object from a string
    return pkg_from_name_size_url(path_or_name)

def pkg_from_name_size_url(path_or_name, size_str=None, url=None): # Create a SlackwarePackage object from a name string and a size string
    matchobj = slackroll_pkg_re.match(path_or_name)

    if matchobj is None:
        raise SlackrollError('nonstandard package name: %s' % path_or_name)

    path = matchobj.group(slackroll_pkg_re_path)
    name = matchobj.group(slackroll_pkg_re_name)
    version = matchobj.group(slackroll_pkg_re_version)
    arch = matchobj.group(slackroll_pkg_re_arch)
    build = matchobj.group(slackroll_pkg_re_build)
    suffix = matchobj.group(slackroll_pkg_re_suffix)
    try:
        size = (size_str is not None and long(size_str) or None)
    except ValueError:
        raise SlackrollError('invalid file size: %s' % size_str)
    return SlackwarePackage(name, version, arch, build, path, suffix, size, url)

def pkg_in_map(pkg, map_by_name): # Checks if pkg is in map[pkg.name], usable for local_list and remote_list
    return pkg in map_by_name.get(pkg.name, [])

class ChangeLogEntry(object):
    _timestamp = None    # Entry timestamp
    _text = None        # Entry text

    def __init__(self, timestamp, text):
        self._timestamp = timestamp
        self._text = text

    @property
    def timestamp(self):
        return self._timestamp

    @property
    def text(self):
        return self._text

    def __eq__(self, other):
        return (self.timestamp == other.timestamp or self.text == other.text)

    def __str__(self):
        return '%s\n%s' % (self.timestamp, self.text)

class ChangeLog(object):
    _cur_batch = None
    _batches = None

    def __init__(self):
        self._cur_batch = 0
        self._batches = dict()

    def start_new_batch(self):
        self._cur_batch += 1

    def add_entry(self, entry):
        val = self._batches.get(self._cur_batch, [])
        val.append(entry)
        self._batches[self._cur_batch] = val

    def add_entries(self, entry_list):
        for x in entry_list:
            self.add_entry(x)

    def last_batch(self):
        return self._batches.get(self._cur_batch, [])

    def num_batches(self):
        return len(self._batches.keys())

    def get_batch(self, batchnum):
        return self._batches.get(batchnum, [])

def clentry_from_text(text):
    lines = text.split('\n')
    text = '\n'.join(lines[1:])
    timestamp = lines[0].strip()
    return ChangeLogEntry(timestamp, text)

def clentrylist_from_text(text):
    entries = text.split(slackroll_changelog_entry_separator)
    entries = [x for x in entries if len(x) > 0]
    return [clentry_from_text(x) for x in entries]

def print_flush(message):
    sys.stdout.write(message)
    sys.stdout.flush()

def concat(list_of_lists):
    return reduce(operator.concat, list_of_lists, [])

def get_env_or(varname, defval):
    value = os.getenv(varname)
    if value is None:
        return defval
    return value

def get_temp_dir(): # Temporary directory name
    return get_env_or('TMPDIR', slackroll_default_temp_dir)

def get_pager():
    return get_env_or('PAGER', slackroll_default_pager)

def get_visual():
    return get_env_or('VISUAL', slackroll_default_visual)

def get_difftool():
    return get_env_or('SRDIFF', slackroll_default_difftool)

def optimum_size_conversion(bytes):
    suffixes = 'bkMGTPEZY'
    if bytes == 0:
        suffidx = 0
    else:
        suffidx = min(int(math.log(bytes, 1024)), len(suffixes) - 1)
    if suffidx == 0:
        return '%s%s' % (bytes, suffixes[suffidx])
    return '%.1f%s' % (float(bytes) / (1024.0**suffidx), suffixes[suffidx])

def enough_fs_resources(numpkgs, bytes): # Check if there are enough filesystem resources to store the given number of packages and bytes
    fs_stats = os.statvfs(os.path.join(slackroll_base_dir, os.path.curdir))
    available_bytes = fs_stats.f_bfree * fs_stats.f_bsize
    return (available_bytes > bytes)

def get_self_file_version(): # Get self version from disk file
    try:
        data = file(slackroll_self_filename, 'r').read()
        return long(data)
    except (OSError, IOError, ValueError), err:
        sys.exit('ERROR: %s' % err)

def write_self_file_version(): # Write self version to disk file
    try:
        file(slackroll_self_filename, 'w').write('%s\n' % slackroll_version)
    except (OSError, IOError), err:
        sys.exit('ERROR: %s' % err)

def long_time_warning():
    print 'WARNING: This operation may take a long time to complete'

def interpret_results_warning():
    print 'WARNING: Results should be interpreted carefully'

def low_fs_resources_warning():
    print 'WARNING: Available disk space may not be enough'

def get_mtime(path):
    try:
        return os.path.getmtime(path)
    except (OSError, IOError), err:
        sys.exit('ERROR: %s' % err)

def newer_than(path1, path2): # path1 modification time more recent than path2 modification time?
    if not os.path.exists(path1) and not os.path.exists(path2):
        return True # Sane default response, according to the usage we make of this function
    if not os.path.exists(path1):
        return False
    if not os.path.exists(path2):
        return True
    reftime = get_mtime(path1)
    modtime = get_mtime(path2)
    return (reftime > modtime)

def is_readable_file(filepath):
    return os.path.isfile(filepath) and os.access(filepath, os.R_OK)

def try_dump(object, filepath): # cPickle.dump()
    try:
        cPickle.dump(object, file(filepath, 'wb'), -1)
    except (OSError, IOError, cPickle.PickleError), err:
        sys.exit('ERROR: %s' % err)

def try_load(filepath): # cPickle.load()
    try:
        return cPickle.load(file(filepath, 'rb'))
    except (OSError, IOError, cPickle.PickleError), err:
        sys.exit('ERROR: %s' % err)

def get_blacklist(): # Get blacklist expression list
    if is_readable_file(slackroll_blacklist_filename):
        return try_load(slackroll_blacklist_filename)
    return []

def split_blacklist_re(blacklist_str):
    if '@' not in blacklist_str:
        return (blacklist_str, '')
    return tuple(blacklist_str.split('@', 1))

def get_blacklist_re(): # Almost the same, but already converted to regular expressions
    regex_list = []
    try:
        for x in get_blacklist():
            pkg_regex, url_regex = split_blacklist_re(x)
            regex_list.append((re.compile(pkg_regex), re.compile(url_regex)))
        return regex_list
    except re.error:
        sys.exit('ERROR: invalid regular expression found in blacklist')

def add_blacklist_exprs(expressions): # Add expressions to the blacklist
    valid_ones = []
    for regex_str in expressions:
        try:
            pkg_regex, url_regex = split_blacklist_re(regex_str)

            re.compile(pkg_regex)
            re.compile(url_regex)
            valid_ones.append(regex_str)
        except re.error:
            sys.exit('ERROR: "%s" is an invalid regular expression' % regex_str)
    bl = get_blacklist()
    bl.extend(valid_ones)
    try_dump(bl, slackroll_blacklist_filename)

def del_blacklist_exprs(indexes): # Remove expressions from the blacklist
    bl = get_blacklist()
    idnums = []
    for idx in indexes:
        try:
            num = long(idx)
            if num < 0 or num >= len(bl):
                raise ValueError()
            idnums.append(num)
        except ValueError:
            sys.exit('ERROR: invalid blacklist entry index: %s' % idx)
    index_regex_pairs = [(x, bl[x]) for x in xrange(len(bl))]
    newbl = [regex for (idx, regex) in index_regex_pairs if idx not in idnums]
    try_dump(newbl, slackroll_blacklist_filename)

def print_blacklist():
    bl = get_blacklist()
    numbered = ['%-4s  %s' % (x, bl[x]) for x in xrange(len(bl))]
    print_list_or(numbered, 'Blacklisted expressions:', 'No blacklisted expressions')

def extract_file_list(filename): # Extracts the file list from a local info file
    try:
        lines = file(filename, 'r').readlines()
        paths = ['/%s' % x.strip().decode('string_escape') for x in lines[lines.index(slackroll_local_pkg_filelist_marker) + 1:]]
        return paths
    except ValueError:
        sys.exit('ERROR: unable to find file list marker in %s' % filename)
    except (OSError, IOError), err:
        sys.exit('ERROR: %s' % err)

def get_pkg_filelists(): # Return a dictionary of filename->filelist from slackroll_local_pkgs_dir
    if newer_than(slackroll_local_pkgs_dir, slackroll_pkg_files_filename):
        try:
            filenames = glob.glob(slackroll_local_pkgs_glob)
        except (OSError, IOError), err:
            sys.exit('ERROR: %s' % err)
        contents = dict()
        for filename in filenames:
            contents[filename] = extract_file_list(filename)
        try_dump(contents, slackroll_pkg_files_filename)
    return try_load(slackroll_pkg_files_filename)

def get_normalized_known_files(): # Return a set of known files for orphan-search
    if newer_than(slackroll_local_pkgs_dir, slackroll_known_files):
        try:
            filenames = glob.glob(slackroll_local_pkgs_glob)
        except (OSError, IOError), err:
            sys.exit('ERROR: %s' % err)
        known_files = set()
        for pkg in filenames:
            for path in extract_file_list(pkg):
                known_files.add(os.path.realpath(path))
        try_dump(known_files, slackroll_known_files)
    return try_load(slackroll_known_files)

def get_local_pkgs(): # Return a list of packages from slackroll_local_pkgs_dir
    local_pkgs = glob.glob(slackroll_local_pkgs_glob)
    if len(local_pkgs) == 0:
        sys.exit('ERROR: could not read list of local packages')
    try:
        local_pkgs = [pkg_from_str(x) for x in local_pkgs]
    except SlackrollError, err:
        sys.exit('ERROR: %s' % err)
    return local_pkgs

def get_local_list(forced_rebuild): # Return list of local packages from cached list, updating it if needed
    if newer_than(slackroll_local_pkgs_dir, slackroll_locallist_filename) or forced_rebuild:
        print 'Rebuilding local package list...'
        dupes = set()
        local_list = dict()
        for pkg in get_local_pkgs():
            name = pkg.name
            if name in local_list:
                dupes.add(name)
            value = local_list.get(name, [])
            value.append(pkg)
            local_list[name] = value
        if len(dupes) > 0:
            print 'WARNING: packages with two or more local versions should be frozen or foreign'
            print_seq(dupes, 'WARNING: list of packages with two or more local versions:')
        try_dump(local_list, slackroll_locallist_filename)
    return try_load(slackroll_locallist_filename)

def get_remote_pkgs(local_filelist, url): # Return a list of packages from FILELIST.TXT
    try:
        lines = file(local_filelist, 'r').readlines()
        matches = [slackroll_filelist_pkg_re.match(x) for x in lines]
        nm_sz = [(x.group(slackroll_filelist_pkg_str), x.group(slackroll_filelist_pkg_size)) for x in matches if x is not None]
        return [pkg_from_name_size_url(nm, sz, url) for (nm, sz) in nm_sz if slackroll_source_indicator not in nm]
    except (OSError, IOError), err:
        sys.exit('ERROR: %s' % err)

def extend_remote_list(local_filelist, remote_list, pkgs_url=None):
    bl = get_blacklist_re()
    has_patch = set()
    for pkg in get_remote_pkgs(local_filelist, pkgs_url):
        # Ignore packages matching blacklist
        fullname = pkg.fullname
        url = pkg.url(pkgs_url)
        # If the package full name and its URL match any entry in the blacklist
        if any((pkg_regex.search(fullname) is not None and url_regex.search(url) is not None) for (pkg_regex, url_regex) in bl):
            continue
        # Mark packages with patches
        name = pkg.name
        if slackroll_patch_indicator in pkg.path:
            has_patch.add(name)
        value = remote_list.get(name, [])
        value.append(pkg)
        remote_list[name] = value
    # Remove unpatched versions
    for name in has_patch:
        if any_in_main_tree(remote_list[name]) and any_in_extra_tree(remote_list[name]):
            remote_list[name] = not_main(remote_list[name])
        else:
            remote_list[name] = not_main_or_extra(remote_list[name])

def get_remote_list(): # Return list of remote packages from cached list, updating if needed
    return try_load(slackroll_remotelist_filename)

def get_mirror_from_file(filepath):
    try:
        filename = os.path.basename(filepath)
        lines = file(filepath, 'r').readlines()
        if len(lines) != 1:
            raise ValueError('more than one line on %s file' % filename)
        mirror = lines[0].strip()
        if len(mirror) == 0:
            raise ValueError('%s is empty' % filename)
        if not mirror.endswith('/'):
            mirror = mirror + '/'
        return mirror
    except (OSError, IOError, ValueError), err:
        sys.exit('ERROR: %s' % err)

def get_mirror(): # From the 'mirror' file
    return get_mirror_from_file(slackroll_mirror_filename)

def get_primary_mirror(): # From the 'pmirror' file or default value
    if is_readable_file(slackroll_primary_mirror_filename):
        return get_mirror_from_file(slackroll_primary_mirror_filename)

    arch, version = get_mirror_version_components(get_mirror())

    return get_default_primary_mirror(arch, version)

def get_default_primary_mirror(arch, version):
    if arch == 'arm':
        return slackroll_arm_primary_site_url % (arch, version)
    return slackroll_default_primary_site_url % (arch, version)

def set_mirror(mirror): # Writes the mirror name to the 'mirror' file
    if not mirror.endswith('/'):
        mirror = mirror + '/'
    try:
        file(slackroll_mirror_filename, 'w').write('%s\n' % mirror)
    except (OSError, IOError, ValueError), err:
        sys.exit('ERROR: %s' % err)

def set_primary_mirror(mirror): # Writes the mirror name to the 'pmirror' file
    if not mirror.endswith('/'):
        mirror = mirror + '/'
    try:
        file(slackroll_primary_mirror_filename, 'w').write('%s\n' % mirror)
    except (OSError, IOError, ValueError), err:
        sys.exit('ERROR: %s' % err)

def get_mirror_version_components(mirror): # Extract Slackware version components from mirror URL
    match = slackroll_mirror_version_re.search(mirror)
    if match is None:
        sys.exit('ERROR: unable to extract Slackware version from mirror name')
    return (match.group(1), match.group(2))

def get_repo_list():
    if not is_readable_file(slackroll_repolist_filename):
        repo_list = []
        try_dump(repo_list, slackroll_repolist_filename)
    return try_load(slackroll_repolist_filename)

def dump_repo_list(repo_list):
    try_dump(repo_list, slackroll_repolist_filename)

def get_pkg_cache_size(): # Return total bytes for files in cache
    try:
        return sum(os.path.getsize(x) for x in glob.glob(slackroll_pkgs_dir_glob))
    except (OSError, IOError), err:
        sys.exit('ERROR: %s' % err)

def call_pager(): # Call pager and return handler
    try:
        pager = get_pager()
        pager_list = pager.split()
        proc = subprocess.Popen(pager_list, stdin=subprocess.PIPE)
        return proc
    except (OSError, IOError), err:
        sys.exit('ERROR: %s' % err)

def getwinsize(): # Returns terminal (rows, columns)
    # XXX - Weird code
    #
    # This code needs to invoke the TIOCGWINSZ ioctl and read 4 unsigned
    # short values that would be written to a struct. In C, you pass
    # the struct pointer as an argument to ioctl() and read the struct
    # members after the call, but in Python we need to pass a string
    # buffer that will be copied and modified by the underlying call,
    # and then returned. This string needs to be parsed and interpreted
    # into 4 values. The struct module eases this task. The 4 variables
    # are given the same name they have in the C winsize structure.

    bufsize = struct.calcsize('HHHH')
    bytes = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, '\x00' * bufsize)
    (ws_row, ws_col, ws_xpixel, ws_ypixel) = struct.unpack('HHHH', bytes)
    return (ws_row, ws_col)

def needs_pager(numlines): # Returns True if lines do not fit in screen
    if (not sys.stdout.isatty()) or slackroll_batch_mode:
        return False
    (rows, cols) = getwinsize()
    return (numlines >= rows - 2)

class SlackrollOutputInterceptor(object): # Intercepts stdout and uses a pager if the output is too long
    __was_not_tty = None
    __old_stdout = None
    __buffer = None

    def __init__(self):
        if not sys.stdout.isatty():
            self.__was_not_tty = True
            return
        self.__buffer = cStringIO.StringIO()
        self.__old_stdout = sys.stdout
        sys.stdout = self.__buffer

    def stop(self):
        if self.__was_not_tty:
            return
        sys.stdout = self.__old_stdout
        self.__old_stdout = None
        output = self.__buffer.getvalue()
        self.__buffer.close()
        numlines = output.count('\n')
        if needs_pager(numlines):
            proc = call_pager()
            proc.stdin.write(output)
            proc.stdin.close()
            proc.wait()
        else:
            sys.stdout.write(output)

def run_program(arg_list, env=None): # Does not exit on errors
    try:
        subprocess.call(arg_list, env=env)
    except (OSError, IOError), err:
        sys.stderr.write('ERROR: %s\n' % err)

def run_visual_on(file_path): # Does not exit on errors
    run_program(get_visual().split() + [file_path])

def run_difftool_on(oldfile, newfile): # Does not exit on errors
    run_program(get_difftool().split() + [oldfile, newfile])

def try_to_remove(file_path, fatal=True): # When it returns, it is True if file does not exist on exit
    if not os.path.exists(file_path):
        return True
    try:
        os.remove(file_path)
        return True
    except (OSError, IOError), err:
        sys.stderr.write('ERROR: %s\n' % err)
        if fatal:
            sys.exit(slackroll_exit_failure)
        return False

def try_to_rename(src, dest, fatal=True): # When it returns, it is True if file could be renamed
    try:
        os.rename(src, dest)
        return True
    except (OSError, IOError), err:
        sys.stderr.write('ERROR: %s\n' % err)
        if fatal:
            sys.exit(slackroll_exit_failure)
        return False

def download_report_hook(filename, numblocks, blocksize, totalsize): # Template for urllib.urlretrieve() callbacks
    if totalsize == -1:
        percent = 'N/A'
        size = 'N/A'
    else:
        percent = min(int(round(numblocks * blocksize * 100.0 / totalsize)), 100)
        size = optimum_size_conversion(totalsize)
    print_flush('\rDownloading %s ... %s%% of %s' % (filename, percent, size))

def download_file(mirror, filepath, local_temp, local_final, displayed_name=None): # Does not exit on errors and shouldn't be called directly
    try:
        if displayed_name is None:
            displayed_name = os.path.basename(filepath)
        print_flush('Downloading %s ... ' % displayed_name)
        full_url = urlparse.urljoin(mirror, filepath)
        hook = None if slackroll_batch_mode else functools.partial(download_report_hook, displayed_name)
        urllib.urlretrieve(full_url, local_temp, hook)
        print
        if not try_to_rename(local_temp, local_final, fatal=False):
            raise SlackrollError('rename error: %s => %s' % (local_temp, local_final))
    except (OSError, IOError, socket.error, urllib.ContentTooShortError), err:
        sys.stderr.write('\nERROR: %s\n' % err)
        raise SlackrollError(str(err))

def download(mirror, filepath, localdir, displayed_name=None): # Wrapper that does NOT exit on errors
    name = os.path.basename(filepath)
    localtemp = os.path.join(get_temp_dir(), '%s%s' % (name, slackroll_temp_suffix))
    localfinal = os.path.join(localdir, name)
    download_file(mirror, filepath, localtemp, localfinal, displayed_name)

def download_or_exit(mirror, filepath, localdir, displayed_name=None): # Wrapper that exits on errors
    try:
        download(mirror, filepath, localdir, displayed_name)
    except SlackrollError:
        sys.exit(slackroll_exit_failure)

def handle_writable_dir(dirname): # Make sure dirname is available and writable
    if not os.path.exists(dirname):
        try:
            os.mkdir(dirname)
        except (OSError, IOError), err:
            sys.exit('ERROR: %s' % err)
    if not os.path.isdir(dirname):
        sys.exit('ERROR: %s exists but is not a directory' % dirname)
    if not os.access(dirname, os.R_OK | os.W_OK):
        sys.exit('ERROR: directory %s is not available for read and writing (are you root?)' % dirname)

def yield_gnupg_exec_name():
    for opt in slackroll_gnupg_exec_names:
        try:
            retcode = subprocess.call([opt, '--version'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
            if retcode != 0:
                continue
            while True:
                yield opt
        except (OSError, IOError):
            pass
    sys.exit('ERROR: unable to run GnuPG')

gnupg_exec_name = yield_gnupg_exec_name().next

def import_key(filename):
    try:
        print 'Importing keys from %s ...' % filename
        retcode = subprocess.call([gnupg_exec_name(), '--import', filename], stdout=file(os.path.devnull, 'w'), stderr=subprocess.STDOUT)
        if retcode != 0:
            raise SlackrollError('GnuPG exited with error when importing key')
    except (OSError, IOError, SlackrollError), err:
        sys.exit('ERROR: %s' % err)

def verify_signature(filename):    # Does not exit on errors
    try:
        print 'Verifying signature %s ... ' % os.path.basename(filename)
        retcode = subprocess.call([gnupg_exec_name(), '--verify', filename], stdout=file(os.path.devnull, 'w'), stderr=subprocess.STDOUT)
        if retcode != 0:
            sys.stderr.write('ERROR: signature verification failed: %s\n' % filename)
            raise SlackrollError('GnuPG exited with status code %s' % retcode)
    except (OSError, IOError), err:
        sys.stderr.write('ERROR: %s\n' % err)
        raise SlackrollError(str(err))

def upgrade_or_install(filename, reinstall): # upgradepkg
    try:
        print 'Installing %s ...' % os.path.basename(filename)
        retcode = subprocess.call(['/sbin/upgradepkg', '--install-new'] + ([[], ['--reinstall']][reinstall]) + [filename])
        if retcode != 0:
            sys.exit('ERROR: installation failed: %s' % filename)
    except (OSError, IOError), err:
        sys.exit('ERROR: %s' % err)

def install_with_installpkg(filename): # installpkg
    try:
        print 'Installing %s ...' % os.path.basename(filename)
        retcode = subprocess.call(['/sbin/installpkg', filename])
        if retcode != 0:
            sys.exit('ERROR: installation failed: %s' % filename)
    except (OSError, IOError), err:
        sys.exit('ERROR: %s' % err)

def replace_pkg(installed_filename, new_filename): # upgradepkg using '%' notation
    try:
        print 'Installing %s ...' % os.path.basename(new_filename)
        retcode = subprocess.call(['/sbin/upgradepkg', '%s%%%s' % (installed_filename, new_filename)])
        if retcode != 0:
            sys.exit('ERROR: installation failed: %s' % new_filename)
    except (OSError, IOError), err:
        sys.exit('ERROR %s' % err)

def remove_pkg(removepkg_arg): # removepkg
    try:
        print 'Removing %s ...' % removepkg_arg
        retcode = subprocess.call(['/sbin/removepkg', removepkg_arg])
        if retcode != 0:
            sys.exit('ERROR: removal failed: %s' % removepkg_arg)
    except (OSError, IOError), err:
        sys.exit('ERROR: %s' % err)

def remove_pkgs(pkg_list):
    dotnew_files = extract_dotnew_files(pkg_list)
    for pkg in pkg_list:
        remove_pkg(pkg.idname)
    handle_dotnew_files_removal(dotnew_files)

def package_in_cache(package): # Check if package is in ./packages
    filepath = os.path.join(slackroll_pkgs_dir, package.archivename)
    return is_readable_file(filepath)

def download_verify(mirror, package): # Download package, signature and verify it
    sigmirror = get_primary_mirror()
    tempdir = get_temp_dir()
    remote_name = package.fullname
    remote_sig = package.fullsigname
    local_name = os.path.join(tempdir, package.archivename)
    local_sig = os.path.join(tempdir, package.signame)
    local_final = os.path.join(slackroll_pkgs_dir, package.archivename)

    try:
        download(package.base_url(sigmirror), remote_sig, tempdir)
        download(package.base_url(mirror), remote_name, tempdir)
        verify_signature(local_sig)
    except SlackrollError:
        try_to_remove(local_sig)
        try_to_remove(local_name)
        return None

    try_to_remove(local_sig)
    try_to_rename(local_name, local_final)
    return local_final

def get_remote_info(mirror, package): # Downloads info file and returns its contents
    remote_file = package.fullname[:-len(slackroll_info_suffix)] + slackroll_info_suffix
    local_file = os.path.join(slackroll_base_dir, os.path.basename(remote_file))
    download_or_exit(package.base_url(mirror), remote_file, slackroll_base_dir)
    data = file(local_file, 'r').read()
    try_to_remove(local_file)
    return data

def update_changelog(mirror, full=False): # Returns answer to "has it been updated?"
    if not os.path.exists(slackroll_local_changelog) or full:
        temp_dir = get_temp_dir()
        download_or_exit(mirror, slackroll_changelog_filename, temp_dir)
        local_filename = os.path.join(temp_dir, slackroll_changelog_filename)
        text = file(local_filename).read()
        cl = ChangeLog()
        cl.add_entries(clentrylist_from_text(text))
        try_dump(cl, slackroll_local_changelog)
        try_to_remove(local_filename)
        return True

    print_flush('Updating change log ... ')

    try: # Only read until last known entry
        cl = try_load(slackroll_local_changelog)
        limits = set(concat([[x.timestamp for x in cl.get_batch(y)] for y in xrange(cl.num_batches())]))
    except (OSError, IOError), err:
        sys.exit('ERROR: %s' % err)

    changelog_url = urlparse.urljoin(mirror, slackroll_changelog_filename)
    lines = []
    try:
        conn = urllib.urlopen(changelog_url)
        while True:
            new_line = conn.readline()
            if len(new_line) == 0 or new_line.strip() in limits:
                break
            lines.append(new_line)
        conn.close()
    except (OSError, IOError, socket.error, urllib.ContentTooShortError), err:
        sys.exit('\nERROR: %s' % err)

    if len(lines) == 0:
        print 'no new entries.'
        return False
    print 'new entries found.'
    cl.start_new_batch()
    cl.add_entries(clentrylist_from_text(''.join(lines)))
    try_dump(cl, slackroll_local_changelog)
    return True

def print_urls(mirror, package):
    sigmirror = get_primary_mirror()
    pkgurl = package.url(mirror)
    sigurl = package.sig_url(sigmirror)
    print sigurl
    print pkgurl

def choose_option(option_list): # Prompt user and return option index
    print 'Choose option:'
    for (code, text) in option_list:
        print '    (%s) %s' % (code, text)
    codes = [x[0].lower() for x in option_list]
    while True:
        print_flush('You choose option... ')
        chosen = raw_input().lower()
        if chosen not in codes:
            continue
        break
    return codes.index(chosen)

def choose_pkg(pkg_list): # Returns chosen package or None
    if slackroll_batch_mode:
        raise SlackrollBatchModeError('Cannot automatically choose action in batch mode')

    options = [(str(x + 1), pkg_list[x].fullname) for x in xrange(len(pkg_list))]
    chosen = choose_option(options)
    return pkg_list[chosen]

def any_in_main_tree(pkg_list): # True if any package from the list is in the main tree
    return (len([x for x in pkg_list if slackroll_main_indicator.search(x.path) is not None]) > 0)

def any_in_extra_tree(pkg_list): # True if any package from the list is in the extra tree
    return (len([x for x in pkg_list if slackroll_extra_indicator in x.path]) > 0)

def not_pasture(pkg_list): # Returns packages not in pasture
    return [x for x in pkg_list if slackroll_pasture_indicator not in x.path]

def not_main(pkg_list): # Returns packages not in the main tree
    return [x for x in pkg_list if slackroll_main_indicator.search(x.path) is None]

def not_main_or_extra(pkg_list): # Returns packages not in the main or extra trees
    return [x for x in pkg_list if slackroll_main_indicator.search(x.path) is None and slackroll_extra_indicator not in x.path]

def pkgs_in_state(persistent_list, state_list): # Get persistent_list names with matching state
    return [x for x in persistent_list if persistent_list[x] in state_list]

def key_pkg_in(name_list): # Does any name match with a prioritized package?
    return (len(set(slackroll_prioritized_pkgs).intersection(set(name_list))) > 0)

def key_transient_pkgs(persistent_list):
    return [x for x in slackroll_prioritized_pkgs if x in persistent_list and persistent_list[x] in slackroll_transient_states]

def key_pkg_activity_pending(persistent_list): # True if any new, outdated or unavailable package is prioritized
    return (len(key_transient_pkgs(persistent_list)) > 0)

def maybe_print_key_pkg_watchout(persistent_list):
    if key_pkg_activity_pending(persistent_list):
        print '\nWATCH OUT: ACTIVITY IN KEY SYSTEM PACKAGES'
        print 'You can upgrade them using "upgrade-key-packages"\n'
        return True
    return False

def maybe_print_key_pkg_warning(persistent_list):
    if key_pkg_activity_pending(persistent_list):
        print 'WARNING: It seems there is activity in key system packages'
        print 'WARNING: You should probably use "upgrade-key-packages" first'
        return True
    return False

def maybe_print_new_warning(persistent_list):
    if len(pkgs_in_state(persistent_list, [slackroll_state_new])) > 0:
        print 'WARNING: There are new packages'
        return True
    return False

def maybe_print_outdated_warning(persistent_list):
    if len(pkgs_in_state(persistent_list, [slackroll_state_outdated])) > 0:
        print 'WARNING: There are outdated packages'
        return True
    return False

def print_repo_mod_advice():
    print 'Repository list modified. Remember to run "update" and "list-transient".'

def maybe_confirm_continue():
    if slackroll_batch_mode:
        return
    raw_input('Press Ctrl+C to cancel or Enter to continue... ')

def print_in_states(states, persistent_list, header, print_state): # Print package name if its state matches
    keys = pkgs_in_state(persistent_list, states)
    if len(keys) == 0:
        return
    print header
    keys.sort(cmp=pkg_name_cmp)
    if print_state:
        fmt = '    %%-%ss  %%s' % max(len(slackroll_state_strings[x]) for x in slackroll_all_states)
        items = [(slackroll_state_strings[persistent_list[x]], x) for x in keys]
    else:
        fmt = '    %s'
        items = keys
    print '\n'.join(fmt % x for x in items)
    print 'End of list'

def print_in_states_or(states, persistent_list, header, message_on_empty, print_state): # Idem using interceptor and a message if no matches
    keys = pkgs_in_state(persistent_list, states)
    if len(keys) == 0:
        print message_on_empty
        return
    interceptor = SlackrollOutputInterceptor()
    print_in_states(states, persistent_list, header, print_state)
    interceptor.stop()

def print_seq(seq, header): # Print every item in sequence, sorted with pkg_name_cmp()
    if len(seq) == 0:
        return
    print header
    items = [x for x in seq]
    items.sort(cmp=pkg_name_cmp)
    for item in items:
        print '    %s' % item
    print 'End of list'

def print_seq_or(seq, header, message_on_empty): # Idem using interceptor and a message if no items
    if len(seq) == 0:
        print message_on_empty
        return
    interceptor = SlackrollOutputInterceptor()
    print_seq(seq, header)
    interceptor.stop()

def print_list(the_list, header): # Print every list entry, sorted
    if len(the_list) == 0:
        return
    print header
    the_list.sort()
    for entry in the_list:
        print '    %s' % entry
    print 'End of list'

def print_list_or(the_list, header, message_on_emtpy): # Idem using interceptor and a message if empty list
    if len(the_list) == 0:
        print message_on_emtpy
        return
    interceptor = SlackrollOutputInterceptor()
    print_list(the_list, header)
    interceptor.stop()

def tr_pkg_detail(local_list, remote_list, persistent_list, pkg_name): # Returns a string with package details for a transient package
    if persistent_list[pkg_name] != slackroll_state_new:
        return ''
    return ' '.join(x.path for x in remote_list[pkg_name])

def error_unknown_packages(name_list): # Print package names as unknown and exit with error
    sys.stderr.write(''.join('WARNING: %s looks like an unexpected full version\n' % x for x in name_list if may_be_full_version(x)))
    sys.stderr.write('ERROR: The following packages are unknown:\n')
    sys.stderr.write(''.join('ERROR:    %s\n' % x for x in name_list))
    sys.exit(slackroll_exit_failure)

def maybe_error_unknown_packages(name_list, by_name):
    unknown = [x for x in name_list if x not in by_name]
    if len(unknown) > 0:
        error_unknown_packages(unknown)

def from_states_to_state(orig_states, dest_state, persistent_list, pkg_names): # Changes state if it matches
    maybe_error_unknown_packages(pkg_names, persistent_list)
    dest_st_name = slackroll_state_strings[dest_state]
    print 'Marking packages as %s...' % dest_st_name
    for name in pkg_names:
        cur_st = persistent_list[name]
        if cur_st not in orig_states:
            print '%s: cannot change state from %s to %s' % (name, slackroll_state_strings[cur_st], dest_st_name)
            continue
        persistent_list[name] = dest_state
    persistent_list.sync()

def extract_dotnew_files(local_pkg_list, etc_too=False): # Get .new files from multiple packages
    full = concat([[x for x in extract_file_list(pkg.fullname) if x.endswith(slackroll_new_suffix)] for pkg in local_pkg_list])
    if etc_too:
        walk_append_if(slackroll_etc_dir, is_dotnew_file, full)
    return sorted(set(full))

def old_file(filename): # Returns filename without .new suffix
    if filename.endswith(slackroll_new_suffix):
        return filename[:-(len(slackroll_new_suffix))]
    return filename

def handle_dotnew_files_installation(dotnew_files): # Handles .new files present in packages
    if len(dotnew_files) == 0:
        return

    if slackroll_batch_mode:
        handle_dotnew_files_installation_batch(dotnew_files)
    else:
        handle_dotnew_files_installation_interactive(dotnew_files)

def handle_dotnew_files_installation_batch(dotnew_files): # Handles .new files present in packages in batch mode
    # Iterate over every pair of .new files
    for newfile in dotnew_files:
        oldfile = old_file(newfile)

        try:
            new_exists = os.path.exists(newfile)
            old_exists = os.path.exists(oldfile)
        except (OSError, IOError), err:
            sys.stderr.write('ERROR: %s\n' % err)
            break

        if new_exists and old_exists: # Both
            print 'Keeping both %s and %s for manual review' % (newfile, oldfile)
        elif new_exists and not old_exists: # New only
            print 'Renaming %s to %s because %s does not exist' % (newfile, oldfile, oldfile)
        elif not new_exists and old_exists: # Old only
            print 'Ignoring nonexistent %s and keeping %s' % (newfile, oldfile)
        else: # None
            print 'WARNING: neither %s nor %s exist' % (newfile, oldfile)

def handle_dotnew_files_installation_interactive(dotnew_files): # Handles .new files present in packages for interactive usage
    visual = get_visual()
    srdiff = get_difftool()
    existence_text = ['MISSING', 'FOUND  ']

    # Iterate over every pair of .new files
    for newfile in dotnew_files:
        oldfile = old_file(newfile)
        basenew = os.path.basename(newfile)
        baseold = os.path.basename(oldfile)

        # Every possible menu option
        go_next = ('X', 'Review next pair')
        remove_new = ('R', 'rm %s' % basenew)
        rename_new = ('M', 'mv %s %s' % (basenew, baseold))
        run_difftool = ('V', '%s %s %s' % (srdiff, baseold, basenew))
        visual_new = ('V', '%s %s' % (visual, basenew))
        visual_old = ('V', '%s %s' % (visual, baseold))

        # Repeating options menu (break when user chooses to review next file)
        pair_iterations = 0
        while True:
            pair_iterations += 1
            try:
                new_exists = os.path.exists(newfile)
                old_exists = os.path.exists(oldfile)
            except (OSError, IOError), err:
                sys.stderr.write('ERROR: %s\n' % err)
                break

            print '\n%s %s' % (['Reviewing', 'Still reviewing'][pair_iterations > 1], newfile)
            print '    %s  %s' % (existence_text[old_exists], baseold)
            print '    %s  %s' % (existence_text[new_exists], basenew)

            # Different options depending on which files exist
            if new_exists and old_exists: # Both
                options = [go_next, remove_new, rename_new, run_difftool]
                chosen = choose_option(options)
                if chosen == 0:
                    break
                elif chosen == 1:
                    print 'Removing %s ...' % newfile
                    try_to_remove(newfile, fatal=False)
                elif chosen == 2:
                    print 'Renaming %s ...' % newfile
                    try_to_rename(newfile, oldfile, fatal=False)
                else:
                    run_difftool_on(oldfile, newfile)

            elif new_exists and not old_exists: # New only
                options = [go_next, rename_new, visual_new]
                chosen = choose_option(options)
                if chosen == 0:
                    break
                if chosen == 1:
                    print 'Renaming %s ...' % newfile
                    try_to_rename(newfile, oldfile, fatal=False)
                else:
                    run_visual_on(newfile)

            elif not new_exists and old_exists: # Old only
                options = [go_next, visual_old]
                chosen = choose_option(options)
                if chosen == 0:
                    break
                else:
                    run_visual_on(oldfile)
            else: # None
                options = [go_next]
                chosen = choose_option(options)
                break

def handle_dotnew_files_removal(dotnew_files): # Handle .new files left behind by removepkg
    pairs = [[x, old_file(x)] for x in dotnew_files]
    to_be_checked = concat(pairs)
    existing = [x for x in to_be_checked if os.path.islink(x) or os.path.exists(x)]

    if len(existing) == 0:
        return

    print '\nSome previous .new files have been found.'
    print 'Examining list in detail (this may take some seconds) ...'
    knownfiles = concat(get_pkg_filelists().values())
    knownfiles.extend(old_file(x) for x in knownfiles if x.endswith(slackroll_new_suffix))
    existing = list(frozenset(existing) - frozenset(knownfiles))

    if len(existing) == 0:
        print 'All of them were present in other packages.'
        return

    if slackroll_batch_mode:
        handle_dotnew_files_removal_batch(dotnew_files)
    else:
        handle_dotnew_files_removal_interactive(dotnew_files)

def handle_dotnew_files_removal_batch(existing): # Remove .new files left behind by removepkg in batch mode
    for path in existing:
        print 'Removing %s left over after package removal' % path
        try_to_remove(path, fatal=False)

def handle_dotnew_files_removal_interactive(existing): # Prompt the user about .new files left behind by removepkg in interactive mode
    for path in existing:
        print '\nFound %s' % path
        options = [ ('K', 'Keep it'), ('R', 'Remove it') ]
        chosen = choose_option(options)
        if chosen == 1:
            try_to_remove(path, fatal=False)

def handle_dotnew_files_both(prev_dotnew, cur_dotnew): # Handle installation and removal of pairs left behind
    handle_dotnew_files_installation(cur_dotnew)
    handle_dotnew_files_removal(list(frozenset(prev_dotnew) - frozenset(cur_dotnew)))

def may_be_full_version(pkg_string):
    try:
        pkg_from_str(pkg_string)
        return True
    except SlackrollError:
        pass
    return False

def verify_local_names(names, local_list):
    for name in names:
        if name not in local_list:
            if may_be_full_version(name):
                print 'WARNING: %s looks like an unexpected full version' % name
            sys.exit('ERROR: %s is not a local package name' % name)

def post_kernel_operation(): # Operations needed after kernel installation
    if slackroll_batch_mode:
        print 'WARNING: you may need to modify your bootloader configuration and reboot'
        return

    visual = get_visual()

    while True:
        # Add options to edit known bootloader configuration files
        existant = [x for x in slackroll_bootloader_config_files if os.path.exists(x)]
        options = [(str(x + 1), '%s %s' % (visual, existant[x])) for x in xrange(len(existant))]
        actions = dict([(x, functools.partial(run_visual_on, existant[x])) for x in xrange(len(existant))])

        # Add option to run lilo if present
        if os.path.exists(slackroll_lilo_path):
            options.append(('L', slackroll_lilo_path))
            actions[len(options) - 1] = functools.partial(run_program, [slackroll_lilo_path])

        # Add option to drop to shell, with a trivial attempt at setting a
        # prompt. This may not work if the shell does not understand PS1 or if
        # the user sets PS1 from a script like .bashrc.
        shell = pwd.getpwuid(os.geteuid()).pw_shell
        newenv = dict(os.environ)
        newenv['PS1'] = '(slackroll) %s%% ' % (os.path.basename(shell), )
        options.append(('S', 'Shell'))
        actions[len(options) - 1] = functools.partial(run_program, [shell], env=newenv)

        # Add exit option
        options.append(('X', 'Done'))

        # Display menu and run action
        chosen = choose_option(options)
        if chosen == len(options) - 1: # Exit option
            break
        actions[chosen]() # Run selected action
        print

def update_manifest_database():
    if not os.path.exists(slackroll_manifest_list_filename):
        sys.exit('ERROR: %s not found: run "update" first' % slackroll_manifest_list_filename)

    # manifest_list is a list of (mirror_url, file_path) pairs
    manifest_list = try_load(slackroll_manifest_list_filename)
    tmpdir = get_temp_dir()
    output_name = os.path.join(tmpdir, slackroll_manifest_basename)

    # Download and process files
    manifestdb = dict()
    for (index, url, path) in manifest_list:
        # Download file
        displayed_name = '%s from %s' % (path, ['repository %s' % index, 'mirror'][index is None])
        download_or_exit(url, path, get_temp_dir(), displayed_name)

        # Decompress it
        print_flush('Processing manifest... ')
        stream = bz2.BZ2File(output_name, 'r')
        contents = stream.read()
        stream.close()

        # Read the packages and their files
        file_counter = 0
        for mobj in re.finditer(r'(?ms)^\+\+.*?\sPackage:\s+?(\S+?)\n.*?^\+\+=+\n(.*?)\n\n', contents):
            pkgpath = mobj.group(1)
            pkgfiles = mobj.group(2)

            if slackroll_source_indicator in pkgpath:
                continue
            database_value = pkg_from_str(pkgpath).archivename

            # Store files in database
            for y in re.finditer(r'(?m)^\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(\S+)$', pkgfiles):
                filename = '/' + y.group(1)
                values = manifestdb.get(filename, [])
                values.append(database_value)
                manifestdb[filename] = values
                file_counter += 1
        print '%d files' % file_counter

        # Delete local file
        try_to_remove(output_name)

    # Dump database to disk
    print 'Dumping database to disk...'
    try_dump(manifestdb, slackroll_manifest_filename)

def search_manifest_database(regexp):
    manifestdb = try_load(slackroll_manifest_filename)

    results = []
    for k in manifestdb.iterkeys():
        if regexp.search(k) is not None:
            results.append(k)

    reversed = dict()
    for key in results:
        for pkg in manifestdb[key]:
            values = reversed.get(pkg, [])
            values.append(key)
            reversed[pkg] = values

    if len(reversed) == 0:
        print 'No matching files'
    else:
        interceptor = SlackrollOutputInterceptor()
        print 'Matching files:'
        for pkg in reversed:
            print '    %s:' % pkg
            print '\n'.join('\t%s' % x for x in reversed[pkg])
            print
        print 'End of list'
        interceptor.stop()

def parse_pkg_arg(arg): # Returns tuple (Is full name?, Name, SlackwarePackage or None) -- auxiliar of parse_install_args()
    if slackroll_pkg_archive_suffix.search(arg) is not None or os.path.sep in arg:
        try:
            pkg = pkg_from_str(arg)
            name = pkg.name
            return (True, name, pkg)
        except SlackrollError:
            sys.exit('ERROR: unable to parse specific package version: %s' % arg)
    return (False, arg, None)

def parse_install_args(args, local_list, remote_list, use_local_version, use_pasture, filter_dupes): # Return chosen_pkgs -- auxiliar of install_operations_family()
    chosen_pkgs = []
    for arg in args:
        # Decide the type of argument (full name or simple name)
        (is_full, name, pkg) = parse_pkg_arg(arg)

        if is_full:
            # Specific version given
            if use_local_version and pkg_in_map(pkg, local_list):
                chosen_pkgs.append([x for x in local_list[name] if x == pkg][0])
            else:
                try:
                    chosen_pkgs.append(remote_list[name][remote_list[name].index(pkg)])
                except (KeyError, ValueError):
                    sys.exit('ERROR: unable to find remote package %s' % pkg.archivename)
            continue

        # Only generic name given
        if name not in remote_list and ((not use_local_version) or (use_local_version and name not in local_list)):
            if may_be_full_version(name):
                print 'WARNING: file extension may be missing on "%s"' % name
            sys.exit('ERROR: no package named %s' % name)

        # Create candidate list
        candidates = []
        if use_local_version:
            candidates.extend(local_list.get(name, []))
        candidates.extend(x for x in remote_list.get(name, []) if x not in candidates or not filter_dupes)

        if not use_pasture: # Filter /pasture/ candidates if indicated
            candidates = not_pasture(candidates)

        # Choose among the candidates
        if len(candidates) == 0:
            print 'WARNING: %s only present in /pasture/' % name
        elif len(candidates) == 1:
            chosen_pkgs.append(candidates[0])
        else:
            if not use_local_version:
                for pkg in local_list.get(name, []):
                    print 'Local: %s' % pkg.archivename
            chosen = choose_pkg(candidates)
            if chosen is not None:
                chosen_pkgs.append(chosen)
    return chosen_pkgs

def install_operations_family(operation, args, local_list, remote_list, persistent_list, use_pasture=True):
    is_real_install = (operation in ['install', 'reinstall', 'installpkg'])
    use_local_version = (operation == 'info')
    filter_dupes = (operation == 'info')

    # Warn the user on install and reinstall operations if there is key package activity and they have not indicated key packages
    if is_real_install and not key_pkg_in(args) and maybe_print_key_pkg_warning(persistent_list):
        maybe_confirm_continue()

    # Get mirror and parse arguments (package names or full versions)
    mirror = get_mirror()
    chosen_pkgs = parse_install_args(args, local_list, remote_list, use_local_version, use_pasture, filter_dupes)

    # For install and reinstall operations save list of existing .new file pairs
    if is_real_install and operation != 'installpkg':
        prev_dotnew = extract_dotnew_files(concat([local_list[pkg.name] for pkg in chosen_pkgs if pkg.name in local_list]))

    # After selecting the packages, run the requested operation on them
    if operation != 'info':
        print 'Total size: %s' % optimum_size_conversion(sum(x.size for x in chosen_pkgs))
        if operation != 'urls':
            to_be_downloaded = [x for x in chosen_pkgs if not package_in_cache(x)]
            cached_ones = [x for x in chosen_pkgs if x not in to_be_downloaded]

            for pkg in cached_ones:
                print 'Package %s found in cache' % pkg.archivename

            if not enough_fs_resources(len(to_be_downloaded), sum(x.size for x in to_be_downloaded)):
                low_fs_resources_warning()
                if slackroll_batch_mode:
                    raise SlackrollBatchModeError('Aborting %s in batch mode since fs space is low' % operation)
                maybe_confirm_continue()

            local_pkgs = [os.path.join(slackroll_pkgs_dir, x.archivename) for x in cached_ones]
            for pkg in to_be_downloaded:
                local_name = download_verify(mirror, pkg)
                if local_name is None:
                    sys.exit(slackroll_exit_failure)
                local_pkgs.append(local_name)
            if is_real_install:
                reinstall = (operation == 'reinstall')
                for pkg in local_pkgs:
                    if operation == 'installpkg':
                        install_with_installpkg(pkg)
                    else:
                        upgrade_or_install(pkg, reinstall)
        else:    # urls operation
            for pkg in chosen_pkgs:
                print_urls(mirror, pkg)

    else:    # info operation
        info_map = dict()
        for pkg in chosen_pkgs:
            try:
                if pkg_in_map(pkg, local_list):
                    lines = file(pkg.fullname, 'r').readlines()
                    header = lines[:lines.index(slackroll_local_pkg_filelist_marker)]
                    info_map[pkg.fullname] = ''.join(header)
                else:
                    info_map[pkg.fullname] = get_remote_info(mirror, pkg)
            except (OSError, IOError), err:
                sys.exit('ERROR: %s' % err)
        interceptor = SlackrollOutputInterceptor()
        for pkg in chosen_pkgs:
            print info_map[pkg.fullname]
        interceptor.stop()

    # Review installed .new files
    if is_real_install:
        cur_dotnew = extract_dotnew_files([x.local() for x in chosen_pkgs], etc_too=True)
        if operation == 'installpkg':
            handle_dotnew_files_installation(cur_dotnew)
        else:
            handle_dotnew_files_both(prev_dotnew, cur_dotnew)

def extend_manifest_list(manifest_list, index, mirror, local_filelist):
    # Build the list of MANIFEST.bz2 files
    try:
        lines = open(local_filelist, 'r').readlines()
    except (IOError, OSError), err:
        sys.exit('ERROR: %s' % err)
    files = [slackroll_manifest_re.search(x) for x in lines]
    files = [x.group(1) for x in files if x is not None]
    files = [x for x in files if slackroll_source_indicator not in x]
    new_elements = [(index, mirror, x) for x in files]
    manifest_list.extend(new_elements)

def update_operation(mirror):
    remote_list = dict()
    manifest_list = []
    local_filelist = os.path.join(get_temp_dir(), slackroll_filelist_filename)

    displayed_name = '%s from mirror' % slackroll_filelist_filename
    download_or_exit(mirror, slackroll_filelist_filename, get_temp_dir(), displayed_name)
    extend_remote_list(local_filelist, remote_list)
    extend_manifest_list(manifest_list, None, mirror, local_filelist)
    try_to_remove(local_filelist)

    for (x, repo) in enumerate(get_repo_list()):
        displayed_name = '%s from repository %s' % (slackroll_filelist_filename, x)
        full_url = urlparse.urljoin(repo, slackroll_filelist_filename)
        download_or_exit(repo, slackroll_filelist_filename, get_temp_dir(), displayed_name)
        extend_remote_list(local_filelist, remote_list, repo)
        extend_manifest_list(manifest_list, x, repo, local_filelist)
        try_to_remove(local_filelist)

    try_dump(remote_list, slackroll_remotelist_filename)
    try_dump(manifest_list, slackroll_manifest_list_filename)

    if os.path.exists(slackroll_local_filelist):
        # Remove old file that may have been left behind by old program versions
        try_to_remove(slackroll_local_filelist)

def walk_append_if(root, condition, output): # Walk with os.walk() and append if entries make condition true
    abspath = os.path.abspath(root)
    for (dirpath, dirnames, filenames) in os.walk(abspath):
        for sub in dirnames + filenames:
            full = os.path.join(dirpath, sub)
            if condition(full):
                output.append(full)

def is_not_link(path):
    return not os.path.islink(path)

def is_broken_link(path):
    return os.path.islink(path) and not os.path.exists(path)

def is_dotnew_file(path):
    return os.path.isfile(path) and not os.path.islink(path) and path.endswith(slackroll_new_suffix)

def up_to_date(local_versions, remote_versions): # Is the local package up to date?
    return (len([x for x in local_versions if x in remote_versions]) > 0)

def outdated_or_installed(local_versions, remote_versions): # Returns outdated or installed state depending on up_to_date()
    return [slackroll_state_outdated, slackroll_state_installed][up_to_date(local_versions, remote_versions)]

def analyze_changes(local_list, remote_list, persistent_list): # XXX THIS FUNCTION IS A CENTRAL PIECE OF CODE
    print 'Updating persistent database...'
    already_analyzed = set()

    # Go over packages present in local system and update their state or introduce them
    for name in local_list:
        already_analyzed.add(name)
        if name in persistent_list:
            # Update their state if needed
            state = persistent_list[name]
            if state in [slackroll_state_new, slackroll_state_notinstalled]:
                if name in remote_list:
                    persistent_list[name] = outdated_or_installed(local_list[name], not_pasture(remote_list[name]))
                else:
                    persistent_list[name] = slackroll_state_unavailable
            elif state in [slackroll_state_unavailable, slackroll_state_foreign]:
                if name in remote_list:
                    persistent_list[name] = outdated_or_installed(local_list[name], not_pasture(remote_list[name]))
            elif state in [slackroll_state_frozen]:
                if name not in remote_list:
                    persistent_list[name] = slackroll_state_unavailable
            elif state in [slackroll_state_installed, slackroll_state_outdated]:
                if name not in remote_list:
                    persistent_list[name] = slackroll_state_unavailable
                else:
                    persistent_list[name] = outdated_or_installed(local_list[name], not_pasture(remote_list[name]))

        else:
            # Introduce them in the persistent list
            if name in remote_list:
                persistent_list[name] = outdated_or_installed(local_list[name], not_pasture(remote_list[name]))
            else:
                persistent_list[name] = slackroll_state_unavailable

    # Go over remaining remote packages not already analyzed (hence, not present in local system)
    for name in remote_list:
        if name in already_analyzed:
            continue
        already_analyzed.add(name)
        if name in persistent_list:
            # Update their state if needed
            state = persistent_list[name]
            if state in [slackroll_state_unavailable]:
                persistent_list[name] = slackroll_state_new
            elif state in [slackroll_state_installed, slackroll_state_outdated, slackroll_state_foreign, slackroll_state_frozen]:
                persistent_list[name] = slackroll_state_notinstalled
        else:
            # Introduce them as new unless they are only present in /pasture/
            if len(not_pasture(remote_list[name])) == 0:
                persistent_list[name] = slackroll_state_notinstalled
            else:
                persistent_list[name] = slackroll_state_new

    # Remaining packages, not present in local or remote systems, need to disappear
    for name in persistent_list.keys():    # Must get keys before, as we are going to delete entries
        if name in already_analyzed:
            continue
        del persistent_list[name]

    # Sync database to disk
    persistent_list.sync()

def verify_operation_and_args(op_num_args, operation, args): # Verify number of arguments
    if operation not in op_num_args:
        # Find the operations that the user may be meaning.
        given_words = operation.split('-')
        command_words = [tuple(x.split('-')) for x in op_num_args.keys()]
        distances = dict((x, words_to_words_distance(given_words, x)) for x in command_words)
        min_distance = min(distances.values())
        best_matches = [x for x in distances if distances[x] == min_distance]
        best_matches.sort()

        # Print the error and the list.
        sys.stderr.write('ERROR: no operation called "%s"\n' % (operation, ))
        sys.stderr.write('Use the "help" operation to get a list.\n')
        sys.stderr.write('Did you mean %s?\n' % ' or '.join('"%s"' % ('-'.join(x), ) for x in best_matches))
        sys.exit(slackroll_exit_failure)
    else:
        verify_num_args(op_num_args[operation], operation, args)

def verify_num_args(num, opname, arg_list): # Verify number of arguments of a given operation
    # -1 means any nonzero quantity
    # -2 means zero or one
    if num == -2:
        if len(arg_list) > 1:
            sys.exit('ERROR: %s expects one argument or no arguments' % opname)
    elif num == -1:
        if len(arg_list) == 0:
            sys.exit('ERROR: %s expects more arguments' % opname)
    else:
        if len(arg_list) != num:
            sys.exit('ERROR: %s expects %s argument%s' % (opname, ['no', str(num)][num > 0], ['s', ''][num == 1]))

def levenshtein_distance(word1, word2): # Levenshtein distance between words.
    l1 = len(word1)
    l2 = len(word2)

    # Distances matrix.
    d = [[0] * (l2+1) for i in xrange(l1+1)]

    # Base cases.
    for i in xrange(l1+1):
        d[i][0] = i
    for j in xrange(l2+1):
        d[0][j] = j

    # Recursive cases in order.
    for j in xrange(1, l2+1):
        for i in xrange(1, l1+1):
            if word1[i-1] == word2[j-1]:
                d[i][j] = d[i-1][j-1]
            else:
                d[i][j] = min(d[i-1][j] + 1, d[i][j-1] + 1, d[i-1][j-1] + 1)

    return d[l1][l2]

def word_to_word_list_distance(w, words): # Distance between word and list of words.
    # We will consider this the minimum distance from any word in the list of
    # words to the subject word.
    return min(levenshtein_distance(w, i) for i in words)

def words_to_words_distance(words1, words2): # Distance between two lists of words.
    # We will consider this the sum of the minimum distances for each word
    # against the words of the other list, plus the difference in words.
    return (sum(word_to_word_list_distance(i, words2) for i in words1) + abs(len(words1) - len(words2)))

### Main program ###
try:
    local_list = None
    remote_list = None
    persistent_list = None
    urllib._urlopener = SlackrollURLopener()
    socket.setdefaulttimeout(slackroll_socket_timeout)
    standarize_locales()

    op_num_args = {    # Map of operations and their appropriate number of arguments.
        'add-repo':                 -1,
        'batch':                    -1,
        'blacklist-add':            -1,
        'blacklist-del':            -1,
        'broken-symlinks':          -1,
        'changelog':                0,
        'changelog-entries':        -1,
        'clean-cache':              0,
        'download':                 -1,
        'download-changelog':       0,
        'download-key-packages':    0,
        'download-new':             0,
        'download-path':            -1,
        'download-upgrades':        0,
        'erase-cache':              0,
        'erase-tmp':                0,
        'erase-all':                0,
        'foreign':                  -1,
        'frozen':                   -1,
        'freeze':                   -1,
        'full-changelog':           0,
        'help':                     -2,
        'import-key':               0,
        'info':                     -1,
        'info-new':                 0,
        'info-path':                -1,
        'install':                  -1,
        'install-foreign':          -1,
        'install-new':              0,
        'install-path':             -1,
        'installed':                -1,
        'kernel-upgrade':           0,
        'kernel-clean':             0,
        'list-all':                 0,
        'list-alternatives':        0,
        'list-changelog':           0,
        'list-foreign':             0,
        'list-frozen':              0,
        'list-installed':           0,
        'list-local':               0,
        'list-new':                 0,
        'list-not-installed':       0,
        'list-outdated':            0,
        'list-outdated-frozen':     0,
        'list-remote':              0,
        'list-repos':               0,
        'list-transient':           0,
        'list-unavailable':         0,
        'list-upgrades':            0,
        'list-versions':            -1,
        'local-info':               -1,
        'local-search':             -1,
        'manifest-search':          -1,
        'missing-search':           0,
        'name-search':              -1,
        'new':                      -1,
        'new-not-installed':        0,
        'not-installed':            -1,
        'orphan-search':            -1,
        'path-search':              -1,
        'print-blacklist':          0,
        'print-mirror':             0,
        'print-primary-mirror':     0,
        'reinstall':                -1,
        'remote-paths':             0,
        'remove':                   -1,
        'remove-path':              -1,
        'remove-repo':              -1,
        'remove-unavailable':       0,
        'replace':                  2,
        'set-mirror':               1,
        'set-primary-mirror':       1,
        'state':                    -1,
        'touch':                    0,
        'unavailable':              -1,
        'unavailable-foreign':      0,
        'unfreeze':                 -1,
        'update':                   0,
        'update-manifest':          0,
        'upgrade':                  0,
        'upgrade-key-packages':     0,
        'urls':                     -1,
        'urls-key-packages':        0,
        'urls-new':                 0,
        'urls-path':                -1,
        'urls-upgrades':            0,
        'version':                  0,
    }

    operation = sys.argv[1]
    args = sys.argv[2:]
    verify_operation_and_args(op_num_args, operation, args)

    if operation == 'batch':
        del op_num_args['batch']
        operation = args[0]
        slackroll_batch_mode = True
        args = args[1:]
        verify_operation_and_args(op_num_args, operation, args)

    handle_writable_dir(slackroll_base_dir)

    if operation == 'help':
        if len(args) == 0:
            helpname = '00-default'
        else:
            helpname = args[0]
        helpfile = slackroll_help_file_template % helpname
        if not is_readable_file(helpfile):
            sys.exit('ERROR: no help found for "%s"' % helpname)
        interceptor = SlackrollOutputInterceptor()
        print file(helpfile, 'r').read()
        interceptor.stop()
        sys.exit()

    if operation == 'version':
        print 'slackroll v%s' % slackroll_version
        sys.exit()

    if operation == 'print-blacklist':
        print_blacklist()
        sys.exit()

    if operation == 'blacklist-add':
        add_blacklist_exprs(args)
        sys.exit()

    if operation == 'blacklist-del':
        del_blacklist_exprs(args)
        sys.exit()

    if operation == 'print-mirror':
        print get_mirror()
        sys.exit()

    if operation == 'print-primary-mirror':
        print get_primary_mirror()
        sys.exit()

    if operation == 'set-mirror':
        set_mirror(args[0])
        sys.exit()

    if operation == 'set-primary-mirror':
        set_primary_mirror(args[0])
        sys.exit()

    if operation == 'add-repo':
        repo_list = get_repo_list()
        for r in args:
            if not r.endswith('/'):
                r = r + '/'
            if r not in repo_list:
                repo_list.append(r)
        dump_repo_list(repo_list)
        print_repo_mod_advice()
        sys.exit()

    if operation == 'list-repos':
        repo_list = get_repo_list()
        formatted_list = ['Number %s: %s' % (i, r) for (i, r) in enumerate(repo_list)]
        print_list_or(formatted_list, 'List of repositories:', 'No third-party repositories found')
        sys.exit()

    if operation == 'remove-repo':
        repo_list = get_repo_list()
        repo_dict = dict(enumerate(repo_list))
        valid = []
        invalid = []
        for x in args:
            try:
                num = long(x)
                if num in repo_dict:
                    valid.append(num)
                else:
                    invalid.append(num)
            except:
                invalid.append(x)
        if len(invalid) > 0:
            sys.exit('ERROR: invalid repository numbers: %s' % ', '.join(str(x) for x in invalid))
        for x in valid:
            del repo_dict[x]
        repo_list = [repo_dict[x] for x in sorted(repo_dict.keys())]
        dump_repo_list(repo_list)
        print_repo_mod_advice()
        sys.exit()

    if operation in ['erase-cache', 'erase-tmp', 'erase-all']:
        directories = []
        if operation in ['erase-cache', 'erase-all']:
            directories.append(slackroll_pkgs_dir)
        if operation in ['erase-tmp', 'erase-all']:
            directories.append(get_temp_dir())
        files = concat([glob.glob(os.path.join(directory, '*')) for directory in directories])
        if len(files) == 0:
            print 'No files to remove'
        else:
            files.sort()
            for filename in files:
                try_to_remove(filename)
        sys.exit()

    if operation == 'changelog':
        interceptor = SlackrollOutputInterceptor()
        cl = try_load(slackroll_local_changelog)
        print slackroll_changelog_entry_separator.join('%s' % entry for entry in cl.last_batch())
        interceptor.stop()
        sys.exit()

    if operation == 'list-changelog':
        interceptor = SlackrollOutputInterceptor()
        cl = try_load(slackroll_local_changelog)
        print 'ChangeLog.txt entries:'
        for idb in xrange(cl.num_batches() - 1, -1, -1):
            batch = cl.get_batch(idb)
            for ide in xrange(len(batch)):
                print '    %-8s  %s' % ('%s.%s' % (idb, ide), batch[ide].timestamp)
        print 'End of list'
        interceptor.stop()
        sys.exit()

    if operation == 'changelog-entries':
        cl = try_load(slackroll_local_changelog)
        entries = []
        for code in args:
            try:
                (idb, ide) = code.split('.')
                (idb, ide) = (long(idb), long(ide))
                entries.append(cl.get_batch(idb)[ide])
            except (IndexError, ValueError):
                sys.exit('ERROR: invalid entry: %s' % code)
        interceptor = SlackrollOutputInterceptor()
        print slackroll_changelog_entry_separator.join('%s' % x for x in entries)
        interceptor.stop()
        sys.exit()

    if operation == 'full-changelog':
        cl = try_load(slackroll_local_changelog)
        entries = concat([[x for x in cl.get_batch(y)] for y in xrange(cl.num_batches() - 1, -1, -1)])
        interceptor = SlackrollOutputInterceptor()
        print slackroll_changelog_entry_separator.join('%s' % x for x in entries)
        interceptor.stop()
        sys.exit()

    if operation in ['local-search', 'missing-search']:
        if operation == 'local-search':
            all_regexes = '|'.join('(?:%s)' % x for x in args)
            try:
                regexp = re.compile(all_regexes)
            except re.error:
                sys.exit('ERROR: invalid regular expression')
            test = lambda x: regexp.search(x) is not None
            header = 'Matching files:'
            empty = 'No matching files found'
        else: # missing-search
            long_time_warning()
            interpret_results_warning()
            maybe_confirm_continue()
            print
            test = lambda x: slackroll_never_missing_re.search(x) is None and not os.path.exists(x) and not os.path.exists(old_file(x))
            header = 'Missing files:'
            empty = 'No missing files'


        print 'Reading contents of %s ...' % slackroll_local_pkgs_dir
        pkg_files_map = get_pkg_filelists()

        matches = [(x, [y for y in pkg_files_map[x] if test(y)]) for x in sorted(pkg_files_map.keys())]
        matches = [(name, files) for (name, files) in matches if len(files) > 0]
        if len(matches) == 0:
            print empty
            sys.exit()

        matches.sort()
        interceptor = SlackrollOutputInterceptor()
        print header
        for (name, files) in matches:
            print '    %s' % os.path.basename(name)
            print ''.join('\t%s\n' % x for x in files)
        print 'End of list'
        interceptor.stop()
        sys.exit()

    if operation == 'orphan-search':
        long_time_warning()
        interpret_results_warning()
        maybe_confirm_continue()

        print '\nReading normalized contents of %s ...' % slackroll_local_pkgs_dir
        known_files = get_normalized_known_files()

        print 'Examining specified paths...'
        tree = []
        for path in args: # Iterate over specified paths
            walk_append_if(path, is_not_link, tree)

        # Detect orphans
        print 'Finding orphan files...'
        orphans = []
        for name in tree:
            alternative = '%s%s' % (name, slackroll_new_suffix)
            if name not in known_files and alternative not in known_files:
                orphans.append(name)

        # Print results
        print_list_or(orphans, 'Orphan files:', 'No orphan files found')
        sys.exit()

    if operation == 'broken-symlinks':
        interpret_results_warning()
        maybe_confirm_continue()

        print '\nExamining specified paths...'
        broken_symlinks = []
        for path in args:
            walk_append_if(path, is_broken_link, broken_symlinks)

        print_list_or(broken_symlinks, 'Broken symlinks:', 'No broken symlinks found')
        sys.exit()

    if operation == 'manifest-search':
        all_regexes = '|'.join('(?:%s)' % x for x in args)
        try:
            regexp = re.compile(all_regexes)
        except re.error:
            sys.exit('ERROR: invalid regular expression')
        search_manifest_database(regexp)
        sys.exit()

    # Some operations below need to have the temporary directory available to download stuff
    handle_writable_dir(get_temp_dir())

    if operation == 'import-key':
        download_or_exit(get_primary_mirror(), slackroll_gpgkey_filename, slackroll_base_dir)
        import_key(slackroll_local_gpgkey)
        try_to_remove(slackroll_local_gpgkey)
        sys.exit()

    if operation == 'download-changelog':
        update_changelog(get_mirror(), full=True)
        sys.exit()

    if operation == 'update-manifest':
        update_manifest_database()
        sys.exit()

    # Every operation below needs an updated persistent database
    handle_writable_dir(slackroll_pkgs_dir)

    needs_rebuild = (not os.path.exists(slackroll_self_filename) or get_self_file_version() != slackroll_version)
    if needs_rebuild or operation == 'update':
        update_operation(get_mirror())
    local_list = get_local_list(needs_rebuild)
    remote_list = get_remote_list()

    needs_analyze = (newer_than(slackroll_remotelist_filename, slackroll_persistentlist_filename)
            or newer_than(slackroll_locallist_filename, slackroll_persistentlist_filename)
            or operation == 'touch' or needs_rebuild)

    try:
        persistent_list = shelve.open(slackroll_persistentlist_filename, 'c')
    except anydbm.error:
        sys.exit('ERROR: unable to properly open %s' % slackroll_persistentlist_filename)

    if needs_analyze:
        analyze_changes(local_list, remote_list, persistent_list)
        os.utime(slackroll_persistentlist_filename, None)

    write_self_file_version()

    if operation == 'update':
        mirror = get_mirror()
        print 'Package cache size: %s' % optimum_size_conversion(get_pkg_cache_size())
        updated = update_changelog(mirror)
        transient_packages_present = any(persistent_list[x] in slackroll_transient_states for x in persistent_list)
        if updated or transient_packages_present:
            print
            if updated:
                print 'New change log entries found. Use "changelog" to read them.'
            if transient_packages_present:
                print 'Found packages in transient states. Use "list-transient" to list them.'
        sys.exit()

    if operation == 'touch':
        sys.exit()

    if operation == 'remote-paths':
        every_remote_path = concat([[y.fullname for y in remote_list[x]] for x in remote_list])
        every_remote_path.sort()
        interceptor = SlackrollOutputInterceptor()
        for path in every_remote_path:
            print path
        interceptor.stop()
        sys.exit()

    if operation == 'state':
        maybe_error_unknown_packages(args, persistent_list)
        plist_subset = dict([(x, persistent_list[x]) for x in args])
        print_in_states_or(slackroll_all_states, plist_subset, 'Package states:', 'INTERNAL ERROR: no packages specified', True)
        sys.exit()

    if operation in ['upgrade', 'download-upgrades', 'urls-upgrades']:
        if operation == 'upgrade' and maybe_print_new_warning(persistent_list):
            maybe_confirm_continue()

        names = pkgs_in_state(persistent_list, [slackroll_state_outdated])
        names.sort(cmp=pkg_name_cmp)

        if len(names) == 0:
            print 'No outdated packages'
            sys.exit()

        opmap = { 'upgrade': 'install', 'download-upgrades': 'download', 'urls-upgrades': 'urls' }
        install_operations_family(opmap[operation], names, local_list, remote_list, persistent_list, use_pasture=False)
        sys.exit()

    if operation in ['upgrade-key-packages', 'download-key-packages', 'urls-key-packages']:
        names = [x for x in slackroll_prioritized_pkgs if x in persistent_list and persistent_list[x] == slackroll_state_outdated]
        if len(names) == 0:
            print 'No outdated key system packages'
            sys.exit()
        opmap = { 'upgrade-key-packages': 'install', 'download-key-packages': 'download', 'urls-key-packages': 'urls' }
        install_operations_family(opmap[operation], names, local_list, remote_list, persistent_list, use_pasture=False)
        sys.exit()

    if operation == 'kernel-upgrade':
        names = [x for x in local_list if x.startswith(slackroll_kernel_pkg_indicator) and persistent_list[x] == slackroll_state_outdated]
        if len(names) == 0:
            print 'No outdated kernel packages'
        install_operations_family('installpkg', names, local_list, remote_list, persistent_list, use_pasture=False)
        post_kernel_operation()
        sys.exit()

    if operation == 'clean-cache':
        cache_files = glob.glob(slackroll_pkgs_dir_glob)
        cache_pkgs = []
        extraneous_files = []

        cache_files.sort()
        for elem in cache_files:
            try:
                cache_pkgs.append(pkg_from_str(elem))
            except SlackrollError:
                extraneous_files.append(elem)

        to_be_removed = [pkg for pkg in cache_pkgs if not pkg_in_map(pkg, local_list) and not pkg_in_map(pkg, remote_list)]
        if len(to_be_removed) == 0:
            print 'No package files to remove'
        else:
            for pkg in to_be_removed:
                print 'Removing %s ...' % pkg.fullname
                try_to_remove(pkg.fullname)

        if len(extraneous_files) > 0:
            extraneous_files.sort()
            print '\n'.join('WARNING: extraneous file %s' % x for x in extraneous_files)
        sys.exit()

    if operation in ['list-upgrades', 'list-outdated-frozen']:
        if operation == 'list-upgrades':
            names = pkgs_in_state(persistent_list, [slackroll_state_outdated])
        else: # list-outdated-frozen
            names = pkgs_in_state(persistent_list, [slackroll_state_frozen])
            names = [x for x in names if not up_to_date(local_list[x], not_pasture(remote_list[x]))]

        if len(names) == 0:
            if operation == 'list-upgrades':
                print 'No outdated packages'
            else: # list-outdated-frozen
                print 'No frozen packages would be outdated'
            sys.exit()

        interceptor = SlackrollOutputInterceptor()
        if operation == 'list-upgrades':
            print 'Available upgrades:'
        else: # list-outdated-frozen
            print 'Would be outdated:'

        names.sort(cmp=pkg_name_cmp)
        for name in names:
            candidates = not_pasture(remote_list[name])
            if len(candidates) == 0:
                print '    %s: WARNING: only present in /pasture/\n' % name
                continue
            print '    %s:' % name
            print '\n'.join('\tLocal:\t%s' % pkg.archivename for pkg in local_list[name])
            print '\n'.join('\tRemote:\t%s' % pkg.fullname for pkg in candidates)
            print
        print 'End of list'

        if operation == 'list-upgrades':
            maybe_print_key_pkg_watchout(persistent_list)
        interceptor.stop()
        sys.exit()

    if operation == 'list-transient':
        pkgs_and_states = [(pkg, persistent_list[pkg]) for pkg in persistent_list if persistent_list[pkg] in slackroll_transient_states]
        if len(pkgs_and_states) == 0:
            print 'No transient packages found'
            sys.exit()
        pkgs_and_states.sort(cmp=transient_cmp)

        # Translate states to strings and precalculate constants
        pkgs_and_states = [(name, slackroll_state_strings[state]) for (name, state) in pkgs_and_states]
        max_state_len = max(len(slackroll_state_strings[x]) for x in slackroll_transient_states)
        max_name_len = max(len(a) for (a, b) in pkgs_and_states)
        format = '    %%-%ss  %%-%ss  %%s' % (max_state_len, max_name_len)

        # Print the transient list
        interceptor = SlackrollOutputInterceptor()
        print 'Transient packages:'
        print '\n'.join(format % (state, name, tr_pkg_detail(local_list, remote_list, persistent_list, name)) for (name, state) in pkgs_and_states)
        print 'End of list'
        maybe_print_key_pkg_watchout(persistent_list)
        interceptor.stop()
        sys.exit()

    if operation == 'list-new':
        # Very similar to list-transient
        new_pkgs = pkgs_in_state(persistent_list, [slackroll_state_new])
        if len(new_pkgs) > 0:
            max_name_len = max(len(x) for x in new_pkgs)
        else:
            max_name_len = 0
        format = '%%-%ss  %%s' % max_name_len
        pkg_and_details = [format % (x, tr_pkg_detail(local_list, remote_list, persistent_list, x)) for x in new_pkgs]
        print_seq_or(pkg_and_details, 'New packages:', 'No new packages found')
        sys.exit()

    if operation == 'list-alternatives':
        alternatives = dict()
        for name in persistent_list:
            versions = []
            versions.extend(remote_list.get(name, []))
            versions.extend(x for x in local_list.get(name, []) if x not in versions)
            if len(versions) > 1:
                alternatives[name] = versions

        if len(alternatives) == 0:
            print 'No packages with alternative versions'
            sys.exit()

        names = alternatives.keys()
        names.sort(cmp=pkg_name_cmp)
        interceptor = SlackrollOutputInterceptor()
        print 'Packages with alternatives:'
        for name in names:
            print '    %s:' % name
            for ver in alternatives[name]:
                print '\t%s' % ver.fullname
            print
        print 'End of list'
        interceptor.stop()
        sys.exit()

    if operation in ['list-outdated', 'list-unavailable', 'list-installed', 'list-not-installed', 'list-frozen', 'list-foreign']:
        state_str = operation.replace('list-', '')
        header = '%s packages:' % state_str.capitalize()
        empty_message = 'No %s packages found' % state_str
        state = slackroll_state_strings.index(state_str)

        interceptor = SlackrollOutputInterceptor()
        print_in_states_or([state], persistent_list, header, empty_message, False)
        if state in slackroll_transient_states:
            maybe_print_key_pkg_watchout(persistent_list)
        interceptor.stop()
        sys.exit()

    if operation == 'list-local':
        print_seq_or(local_list, 'Local packages:', 'No local packages found')
        sys.exit()

    if operation == 'list-remote':
        print_seq_or(remote_list, 'Remote packages:', 'No remote packages found')
        sys.exit()

    if operation == 'list-all':
        print_seq_or(persistent_list, 'All packages:', 'No packages found')
        sys.exit()

    if operation == 'new-not-installed':
        new_packages = pkgs_in_state(persistent_list, [slackroll_state_new])
        from_states_to_state([slackroll_state_new], slackroll_state_notinstalled, persistent_list, new_packages)
        sys.exit()

    if operation == 'unavailable-foreign':
        unavailable_packages = pkgs_in_state(persistent_list, [slackroll_state_unavailable])
        from_states_to_state([slackroll_state_unavailable], slackroll_state_foreign, persistent_list, unavailable_packages)
        sys.exit()

    if operation in ['frozen', 'freeze']:
        valid_origins = [slackroll_state_frozen, slackroll_state_installed, slackroll_state_outdated]
        from_states_to_state(valid_origins, slackroll_state_frozen, persistent_list, args)
        sys.exit()

    if operation == 'foreign':
        from_states_to_state([slackroll_state_foreign, slackroll_state_unavailable], slackroll_state_foreign, persistent_list, args)
        sys.exit()

    if operation == 'not-installed':
        from_states_to_state([slackroll_state_notinstalled, slackroll_state_new], slackroll_state_notinstalled, persistent_list, args)
        sys.exit()

    if operation == 'unavailable':
        from_states_to_state([slackroll_state_unavailable, slackroll_state_foreign], slackroll_state_unavailable, persistent_list, args)
        sys.exit()

    if operation == 'new':
        from_states_to_state([slackroll_state_new, slackroll_state_notinstalled], slackroll_state_new, persistent_list, args)
        sys.exit()

    if operation in ['installed', 'unfreeze']:
        from_states_to_state([slackroll_state_installed, slackroll_state_frozen], slackroll_state_installed, persistent_list, args)
        analyze_changes(local_list, remote_list, persistent_list) # Forced because it may need to be marked as outdated
        os.utime(slackroll_persistentlist_filename, None)
        sys.exit()

    if operation == 'list-versions':
        maybe_error_unknown_packages(args, persistent_list)
        interceptor = SlackrollOutputInterceptor()
        print 'Available versions:'
        for name in args:
            print '    %s:' % name
            if name in local_list:
                print '\n'.join('\tLocal:\t%s' % ver.archivename for ver in local_list[name])
            if name in remote_list:
                print '\n'.join('\tRemote:\t%s' % ver.fullname for ver in remote_list[name])
            print
        print 'End of list'
        interceptor.stop()
        sys.exit()

    if operation in ['install', 'reinstall', 'download', 'info', 'urls']:
        install_operations_family(operation, args, local_list, remote_list, persistent_list)
        sys.exit()

    if operation in ['install-new', 'download-new', 'info-new', 'urls-new']:
        names = pkgs_in_state(persistent_list, [slackroll_state_new])
        names.sort(cmp=pkg_name_cmp)
        if len(names) == 0:
            print 'No new packages'
            sys.exit()
        opname = operation.replace('-new', '')
        install_operations_family(opname, names, local_list, remote_list, persistent_list)
        sys.exit()

    if operation in ['install-path', 'download-path', 'info-path', 'urls-path', 'remove-path']:
        all_regexes = '|'.join('(?:%s)' % x for x in args)
        try:
            regexp = re.compile(all_regexes)
        except re.error:
            sys.exit('ERROR: invalid regular expression')

        matching_pkgs = [x for x in concat([remote_list[x] for x in remote_list]) if regexp.search(x.path) is not None]

        if operation == 'remove-path': # Filter out packages not present in local system
            matching_pkgs = [x for x in matching_pkgs if pkg_in_map(x, local_list)]

        if len(matching_pkgs) == 0:
            print 'No matching packages'
            sys.exit()

        matching_pkgs.sort()
        if operation == 'remove-path':
            if maybe_print_new_warning(persistent_list) or maybe_print_outdated_warning(persistent_list):
                maybe_confirm_continue()
            remove_pkgs([x.local() for x in matching_pkgs])
        else:
            opname = operation.replace('-path', '')
            archivenames = [x.archivename for x in matching_pkgs]
            install_operations_family(opname, archivenames, local_list, remote_list, persistent_list)

        sys.exit()

    if operation == 'local-info':
        verify_local_names(args, local_list)
        interceptor = SlackrollOutputInterceptor()
        try:
            print '\n'.join(file(x.fullname, 'r').read() for x in concat(local_list[pkg] for pkg in args))
        except (IOError, OSError), err:
            sys.exit('ERROR: %s' % err)
        interceptor.stop()
        sys.exit()

    if operation == 'install-foreign':
        # Verify readable files
        bad_files = [x for x in args if not is_readable_file(x)]
        if len(bad_files) > 0:
            sys.stderr.write('ERROR: the following items are not readable files:\n')
            sys.stderr.write(''.join('ERROR:    %s\n' % x for x in bad_files))
            sys.exit(slackroll_exit_failure)

        # Verify and get package names
        try:
            pkgs = [pkg_from_str(x) for x in args]
        except SlackrollError, err:
            sys.exit('ERROR: %s' % err)

        # The packages must not be present in persistent list or be foreign already
        bad_pkgs = [x for x in pkgs if x.name in persistent_list and persistent_list[x.name] != slackroll_state_foreign]
        if len(bad_pkgs) > 0:
            sys.stderr.write('ERROR: the following packages are known and not foreign\n')
            sys.stderr.write(''.join('ERROR:    %s\n' % x.name for x in bad_pkgs))
            sys.exit(slackroll_exit_failure)

        # Get .new files from old packages
        prev_dotnew = extract_dotnew_files(concat([local_list[x.name] for x in pkgs if x.name in local_list]))

        # Install and mark
        for arg in args:
            upgrade_or_install(arg, False)
        for pkg in pkgs:
            persistent_list[pkg.name] = slackroll_state_foreign
        persistent_list.sync()

        # Go over .new files
        cur_dotnew = extract_dotnew_files([x.local() for x in pkgs], etc_too=True)
        handle_dotnew_files_both(prev_dotnew, cur_dotnew)
        sys.exit()

    if operation == 'replace':
        if maybe_print_key_pkg_warning(persistent_list):
            maybe_confirm_continue()

        # Verify package names
        for name in args:
            if name not in persistent_list:
                if may_be_full_version(name):
                    sys.stderr.write('WARNING: %s looks like an unexpected full version\n' % name)
                sys.exit('ERROR: no package named %s' % name)

        unavpkg = args[0]
        newpkg = args[1]

        # Verify package conditions
        if persistent_list[unavpkg] != slackroll_state_unavailable:
            sys.exit('ERROR: %s must be in the "unavailable" state' % unavpkg)
        if persistent_list[newpkg] != slackroll_state_new:
            sys.exit('ERROR: %s must be in the "new" state' % newpkg)

        # Choose specific package
        if len(remote_list[newpkg]) != 1:
            chosen_new = choose_pkg(remote_list[newpkg])
        else:
            chosen_new = remote_list[newpkg][0]

        # Download
        if not package_in_cache(chosen_new):
            filename_new = download_verify(get_mirror(), chosen_new)
        else:
            filename_new = os.path.join(slackroll_pkgs_dir, chosen_new.archivename)

        # upgradepkg
        prev_dotnew = extract_dotnew_files(local_list[unavpkg])
        replace_pkg(unavpkg, filename_new)
        cur_dotnew = extract_dotnew_files([chosen_new.local()], etc_too=True)
        handle_dotnew_files_both(prev_dotnew, cur_dotnew)

        sys.exit()

    if operation in ['remove', 'remove-unavailable']:
        if maybe_print_new_warning(persistent_list) or maybe_print_outdated_warning(persistent_list):
            maybe_confirm_continue()

        if operation == 'remove':
            verify_local_names(args, local_list)
            final_args = args
        else: # remove-unavailable
            final_args = pkgs_in_state(persistent_list, [slackroll_state_unavailable])

        if len(final_args) == 0:
            print 'No packages to remove'
            sys.exit()

        to_be_removed = []
        if operation == 'remove':
            for name in final_args:
                candidates = local_list[name]
                if len(candidates) == 1:
                    to_be_removed.append(candidates[0])
                    continue
                chosen = choose_pkg(candidates)
                if chosen is not None:
                    to_be_removed.append(chosen)
        else: # remove-unavailable
            for name in final_args:
                to_be_removed.extend(local_list[name])

        remove_pkgs(to_be_removed)
        sys.exit()

    if operation == 'kernel-clean':
        outdated_kernel_pkgs = [[y for y in local_list[x] if y not in remote_list[x]]
                for x in local_list if x.startswith(slackroll_kernel_pkg_indicator) and persistent_list[x] == slackroll_state_installed]
        outdated_kernel_pkgs = concat(outdated_kernel_pkgs)
        if len(outdated_kernel_pkgs) == 0:
            print 'No obsolete kernel packages found'
            sys.exit()
        remove_pkgs(outdated_kernel_pkgs)
        post_kernel_operation()
        sys.exit()

    if operation == 'name-search':
        all_regexes = '|'.join('(?:%s)' % x for x in args)
        try:
            regexp = re.compile(all_regexes)
        except re.error:
            sys.exit('ERROR: invalid regular expression')

        plist_subset = dict([(x, persistent_list[x]) for x in persistent_list if regexp.search(x) is not None])
        print_in_states_or(slackroll_all_states, plist_subset, 'Matching packages:', 'No matching packages found', True)
        sys.exit()

    if operation == 'path-search':
        all_regexes = '|'.join('(?:%s)' % x for x in args)
        try:
            regexp = re.compile(all_regexes)
        except re.error:
            sys.exit('ERROR: invalid regular expression')

        matching_fullnames = [x.fullname for x in concat([remote_list[x] for x in remote_list]) if regexp.search(x.path) is not None]
        print_list_or(matching_fullnames, 'Matching packages:', 'No matching packages found')
        sys.exit()

except (KeyboardInterrupt, IOError, EOFError), error:
    if isinstance(error, KeyboardInterrupt):
        sys.stderr.write('\nProgram aborted by user\n')
    if persistent_list is not None:
        persistent_list.close()
    sys.exit(slackroll_exit_failure)
except SlackrollBatchModeError:
    sys.stderr.write('\nProgram aborted because operation could not be completed in batch mode\n')
    if persistent_list is not None:
        persistent_list.close()
    sys.exit(slackroll_exit_failure)
except IndexError:
    sys.stderr.write('ERROR: no operation given\n')
    sys.stderr.write('Use "help" operation to get a list\n')
    sys.exit(slackroll_exit_failure)
