Application: gvSIG desktop » gvSIG scripting

# swift.py -- Repo implementation atop OpenStack SWIFT

# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>

#

# Author: Fabien Boucher <fabien.boucher@enovance.com>

#

# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU

# General Public License as public by the Free Software Foundation; version 2.0

# or (at your option) any later version. You can redistribute it and/or

# modify it under the terms of either of these two licenses.

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

# You should have received a copy of the licenses; if not, see

# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License

# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache

# License, Version 2.0.

#

"""Repo implementation atop OpenStack SWIFT."""

# TODO: Refactor to share more code with dulwich/repo.py.

# TODO(fbo): Second attempt to _send() must be notified via real log

# TODO(fbo): More logs for operations

import os

import stat

import zlib

import tempfile

import posixpath

try:

    import urlparse

except ImportError:

    import urllib.parse as urlparse

from io import BytesIO

try:

    from ConfigParser import ConfigParser

except ImportError:

    from configparser import ConfigParser

from geventhttpclient import HTTPClient

from dulwich.greenthreads import (

    GreenThreadsMissingObjectFinder,

    GreenThreadsObjectStoreIterator,

    )

from dulwich.lru_cache import LRUSizeCache

from dulwich.objects import (

    Blob,

    Commit,

    Tree,

    Tag,

    S_ISGITLINK,

    )

from dulwich.object_store import (

    PackBasedObjectStore,

    PACKDIR,

    INFODIR,

    )

from dulwich.pack import (

    PackData,

    Pack,

    PackIndexer,

    PackStreamCopier,

    write_pack_header,

    compute_file_sha,

    iter_sha1,

    write_pack_index_v2,

    load_pack_index_file,

    read_pack_header,

    _compute_object_size,

    unpack_object,

    write_pack_object,

    )

from dulwich.protocol import TCP_GIT_PORT

from dulwich.refs import (

    InfoRefsContainer,

    read_info_refs,

    write_info_refs,

    )

from dulwich.repo import (

    BaseRepo,

    OBJECTDIR,

    )

from dulwich.server import (

    Backend,

    TCPGitServer,

    )

try:

    from simplejson import loads as json_loads

    from simplejson import dumps as json_dumps

except ImportError:

    from json import loads as json_loads

    from json import dumps as json_dumps

import sys

"""

# Configuration file sample

[swift]

# Authentication URL (Keystone or Swift)

auth_url = http://127.0.0.1:5000/v2.0

# Authentication version to use

auth_ver = 2

# The tenant and username separated by a semicolon

username = admin;admin

# The user password

password = pass

# The Object storage region to use (auth v2) (Default RegionOne)

region_name = RegionOne

# The Object storage endpoint URL to use (auth v2) (Default internalURL)

endpoint_type = internalURL

# Concurrency to use for parallel tasks (Default 10)

concurrency = 10

# Size of the HTTP pool (Default 10)

http_pool_length = 10

# Timeout delay for HTTP connections (Default 20)

http_timeout = 20

# Chunk size to read from pack (Bytes) (Default 12228)

chunk_length = 12228

# Cache size (MBytes) (Default 20)

cache_length = 20

"""

class PackInfoObjectStoreIterator(GreenThreadsObjectStoreIterator):

    def __len__(self):

        while len(self.finder.objects_to_send):

            for _ in range(0, len(self.finder.objects_to_send)):

                sha = self.finder.next()

                self._shas.append(sha)

        return len(self._shas)

class PackInfoMissingObjectFinder(GreenThreadsMissingObjectFinder):

    def next(self):

        while True:

            if not self.objects_to_send:

                return None

            (sha, name, leaf) = self.objects_to_send.pop()

            if sha not in self.sha_done:

                break

        if not leaf:

            info = self.object_store.pack_info_get(sha)

            if info[0] == Commit.type_num:

                self.add_todo([(info[2], "", False)])

            elif info[0] == Tree.type_num:

                self.add_todo([tuple(i) for i in info[1]])

            elif info[0] == Tag.type_num:

                self.add_todo([(info[1], None, False)])

            if sha in self._tagged:

                self.add_todo([(self._tagged[sha], None, True)])

        self.sha_done.add(sha)

        self.progress("counting objects: %d\r" % len(self.sha_done))

        return (sha, name)

def load_conf(path=None, file=None):

    """Load configuration in global var CONF

    :param path: The path to the configuration file

    :param file: If provided read instead the file like object

    """

    conf = ConfigParser()

    if file:

        conf.readfp(file)

        return conf

    confpath = None

    if not path:

        try:

            confpath = os.environ['DULWICH_SWIFT_CFG']

        except KeyError:

            raise Exception("You need to specify a configuration file")

    else:

        confpath = path

    if not os.path.isfile(confpath):

        raise Exception("Unable to read configuration file %s" % confpath)

    conf.read(confpath)

    return conf

def swift_load_pack_index(scon, filename):

    """Read a pack index file from Swift

    :param scon: a `SwiftConnector` instance

    :param filename: Path to the index file objectise

    :return: a `PackIndexer` instance

    """

    f = scon.get_object(filename)

    try:

        return load_pack_index_file(filename, f)

    finally:

        f.close()

def pack_info_create(pack_data, pack_index):

    pack = Pack.from_objects(pack_data, pack_index)

    info = {}

    for obj in pack.iterobjects():

        # Commit

        if obj.type_num == Commit.type_num:

            info[obj.id] = (obj.type_num, obj.parents, obj.tree)

        # Tree

        elif obj.type_num == Tree.type_num:

            shas = [(s, n, not stat.S_ISDIR(m)) for

                    n, m, s in obj.items() if not S_ISGITLINK(m)]

            info[obj.id] = (obj.type_num, shas)

        # Blob

        elif obj.type_num == Blob.type_num:

            info[obj.id] = None

        # Tag

        elif obj.type_num == Tag.type_num:

            info[obj.id] = (obj.type_num, obj.object[1])

    return zlib.compress(json_dumps(info))

def load_pack_info(filename, scon=None, file=None):

    if not file:

        f = scon.get_object(filename)

    else:

        f = file

    if not f:

        return None

    try:

        return json_loads(zlib.decompress(f.read()))

    finally:

        f.close()

class SwiftException(Exception):

    pass

class SwiftConnector(object):

    """A Connector to swift that manage authentication and errors catching

    """

    def __init__(self, root, conf):

        """ Initialize a SwiftConnector

        :param root: The swift container that will act as Git bare repository

        :param conf: A ConfigParser Object

        """

        self.conf = conf

        self.auth_ver = self.conf.get("swift", "auth_ver")

        if self.auth_ver not in ["1", "2"]:

            raise NotImplementedError(

                "Wrong authentication version use either 1 or 2")

        self.auth_url = self.conf.get("swift", "auth_url")

        self.user = self.conf.get("swift", "username")

        self.password = self.conf.get("swift", "password")

        self.concurrency = self.conf.getint('swift', 'concurrency') or 10

        self.http_timeout = self.conf.getint('swift', 'http_timeout') or 20

        self.http_pool_length = \

            self.conf.getint('swift', 'http_pool_length') or 10

        self.region_name = self.conf.get("swift", "region_name") or "RegionOne"

        self.endpoint_type = \

            self.conf.get("swift", "endpoint_type") or "internalURL"

        self.cache_length = self.conf.getint("swift", "cache_length") or 20

        self.chunk_length = self.conf.getint("swift", "chunk_length") or 12228

        self.root = root

        block_size = 1024 * 12  # 12KB

        if self.auth_ver == "1":

            self.storage_url, self.token = self.swift_auth_v1()

        else:

            self.storage_url, self.token = self.swift_auth_v2()

        token_header = {'X-Auth-Token': str(self.token)}

        self.httpclient = \

            HTTPClient.from_url(str(self.storage_url),

                                concurrency=self.http_pool_length,

                                block_size=block_size,

                                connection_timeout=self.http_timeout,

                                network_timeout=self.http_timeout,

                                headers=token_header)

        self.base_path = str(

            posixpath.join(urlparse.urlparse(self.storage_url).path, self.root))

    def swift_auth_v1(self):

        self.user = self.user.replace(";", ":")

        auth_httpclient = HTTPClient.from_url(

            self.auth_url,

            connection_timeout=self.http_timeout,

            network_timeout=self.http_timeout,

            )

        headers = {'X-Auth-User': self.user,

                   'X-Auth-Key': self.password}

        path = urlparse.urlparse(self.auth_url).path

        ret = auth_httpclient.request('GET', path, headers=headers)

        # Should do something with redirections (301 in my case)

        if ret.status_code < 200 or ret.status_code >= 300:

            raise SwiftException('AUTH v1.0 request failed on ' +

                                 '%s with error code %s (%s)'

                                 % (str(auth_httpclient.get_base_url()) +

                                    path, ret.status_code,

                                    str(ret.items())))

        storage_url = ret['X-Storage-Url']

        token = ret['X-Auth-Token']

        return storage_url, token

    def swift_auth_v2(self):

        self.tenant, self.user = self.user.split(';')

        auth_dict = {}

        auth_dict['auth'] = {'passwordCredentials':

                             {

                                 'username': self.user,

                                 'password': self.password,

                             },

                             'tenantName': self.tenant}

        auth_json = json_dumps(auth_dict)

        headers = {'Content-Type': 'application/json'}

        auth_httpclient = HTTPClient.from_url(

            self.auth_url,

            connection_timeout=self.http_timeout,

            network_timeout=self.http_timeout,

            )

        path = urlparse.urlparse(self.auth_url).path

        if not path.endswith('tokens'):

            path = posixpath.join(path, 'tokens')

        ret = auth_httpclient.request('POST', path,

                                      body=auth_json,

                                      headers=headers)

        if ret.status_code < 200 or ret.status_code >= 300:

            raise SwiftException('AUTH v2.0 request failed on ' +

                                 '%s with error code %s (%s)'

                                 % (str(auth_httpclient.get_base_url()) +

                                    path, ret.status_code,

                                    str(ret.items())))

        auth_ret_json = json_loads(ret.read())

        token = auth_ret_json['access']['token']['id']

        catalogs = auth_ret_json['access']['serviceCatalog']

        object_store = [o_store for o_store in catalogs if

                        o_store['type'] == 'object-store'][0]

        endpoints = object_store['endpoints']

        endpoint = [endp for endp in endpoints if

                    endp["region"] == self.region_name][0]

        return endpoint[self.endpoint_type], token

    def test_root_exists(self):

        """Check that Swift container exist

        :return: True if exist or None it not

        """

        ret = self.httpclient.request('HEAD', self.base_path)

        if ret.status_code == 404:

            return None

        if ret.status_code < 200 or ret.status_code > 300:

            raise SwiftException('HEAD request failed with error code %s'

                                 % ret.status_code)

        return True

    def create_root(self):

        """Create the Swift container

        :raise: `SwiftException` if unable to create

        """

        if not self.test_root_exists():

            ret = self.httpclient.request('PUT', self.base_path)

            if ret.status_code < 200 or ret.status_code > 300:

                raise SwiftException('PUT request failed with error code %s'

                                     % ret.status_code)

    def get_container_objects(self):

        """Retrieve objects list in a container

        :return: A list of dict that describe objects

                 or None if container does not exist

        """

        qs = '?format=json'

        path = self.base_path + qs

        ret = self.httpclient.request('GET', path)

        if ret.status_code == 404:

            return None

        if ret.status_code < 200 or ret.status_code > 300:

            raise SwiftException('GET request failed with error code %s'

                                 % ret.status_code)

        content = ret.read()

        return json_loads(content)

    def get_object_stat(self, name):

        """Retrieve object stat

        :param name: The object name

        :return: A dict that describe the object

                 or None if object does not exist

        """

        path = self.base_path + '/' + name

        ret = self.httpclient.request('HEAD', path)

        if ret.status_code == 404:

            return None

        if ret.status_code < 200 or ret.status_code > 300:

            raise SwiftException('HEAD request failed with error code %s'

                                 % ret.status_code)

        resp_headers = {}

        for header, value in ret.items():

            resp_headers[header.lower()] = value

        return resp_headers

    def put_object(self, name, content):

        """Put an object

        :param name: The object name

        :param content: A file object

        :raise: `SwiftException` if unable to create

        """

        content.seek(0)

        data = content.read()

        path = self.base_path + '/' + name

        headers = {'Content-Length': str(len(data))}

        def _send():

            ret = self.httpclient.request('PUT', path,

                                          body=data,

                                          headers=headers)

            return ret

        try:

            # Sometime got Broken Pipe - Dirty workaround

            ret = _send()

        except Exception:

            # Second attempt work

            ret = _send()

        if ret.status_code < 200 or ret.status_code > 300:

            raise SwiftException('PUT request failed with error code %s'

                                 % ret.status_code)

    def get_object(self, name, range=None):

        """Retrieve an object

        :param name: The object name

        :param range: A string range like "0-10" to

                      retrieve specified bytes in object content

        :return: A file like instance

                 or bytestring if range is specified

        """

        headers = {}

        if range:

            headers['Range'] = 'bytes=%s' % range

        path = self.base_path + '/' + name

        ret = self.httpclient.request('GET', path, headers=headers)

        if ret.status_code == 404:

            return None

        if ret.status_code < 200 or ret.status_code > 300:

            raise SwiftException('GET request failed with error code %s'

                                 % ret.status_code)

        content = ret.read()

        if range:

            return content

        return BytesIO(content)

    def del_object(self, name):

        """Delete an object

        :param name: The object name

        :raise: `SwiftException` if unable to delete

        """

        path = self.base_path + '/' + name

        ret = self.httpclient.request('DELETE', path)

        if ret.status_code < 200 or ret.status_code > 300:

            raise SwiftException('DELETE request failed with error code %s'

                                 % ret.status_code)

    def del_root(self):

        """Delete the root container by removing container content

        :raise: `SwiftException` if unable to delete

        """

        for obj in self.get_container_objects():

            self.del_object(obj['name'])

        ret = self.httpclient.request('DELETE', self.base_path)

        if ret.status_code < 200 or ret.status_code > 300:

            raise SwiftException('DELETE request failed with error code %s'

                                 % ret.status_code)

class SwiftPackReader(object):

    """A SwiftPackReader that mimic read and sync method

    The reader allows to read a specified amount of bytes from

    a given offset of a Swift object. A read offset is kept internaly.

    The reader will read from Swift a specified amount of data to complete

    its internal buffer. chunk_length specifiy the amount of data

    to read from Swift.

    """

    def __init__(self, scon, filename, pack_length):

        """Initialize a SwiftPackReader

        :param scon: a `SwiftConnector` instance

        :param filename: the pack filename

        :param pack_length: The size of the pack object

        """

        self.scon = scon

        self.filename = filename

        self.pack_length = pack_length

        self.offset = 0

        self.base_offset = 0

        self.buff = b''

        self.buff_length = self.scon.chunk_length

    def _read(self, more=False):

        if more:

            self.buff_length = self.buff_length * 2

        l = self.base_offset

        r = min(self.base_offset + self.buff_length, self.pack_length)

        ret = self.scon.get_object(self.filename, range="%s-%s" % (l, r))

        self.buff = ret

    def read(self, length):

        """Read a specified amount of Bytes form the pack object

        :param length: amount of bytes to read

        :return: bytestring

        """

        end = self.offset+length

        if self.base_offset + end > self.pack_length:

            data = self.buff[self.offset:]

            self.offset = end

            return data

        if end > len(self.buff):

            # Need to read more from swift

            self._read(more=True)

            return self.read(length)

        data = self.buff[self.offset:end]

        self.offset = end

        return data

    def seek(self, offset):

        """Seek to a specified offset

        :param offset: the offset to seek to

        """

        self.base_offset = offset

        self._read()

        self.offset = 0

    def read_checksum(self):

        """Read the checksum from the pack

        :return: the checksum bytestring

        """

        return self.scon.get_object(self.filename, range="-20")

class SwiftPackData(PackData):

    """The data contained in a packfile.

    We use the SwiftPackReader to read bytes from packs stored in Swift

    using the Range header feature of Swift.

    """

    def __init__(self, scon, filename):

        """ Initialize a SwiftPackReader

        :param scon: a `SwiftConnector` instance

        :param filename: the pack filename

        """

        self.scon = scon

        self._filename = filename

        self._header_size = 12

        headers = self.scon.get_object_stat(self._filename)

        self.pack_length = int(headers['content-length'])

        pack_reader = SwiftPackReader(self.scon, self._filename,

                                      self.pack_length)

        (version, self._num_objects) = read_pack_header(pack_reader.read)

        self._offset_cache = LRUSizeCache(1024*1024*self.scon.cache_length,

                                          compute_size=_compute_object_size)

        self.pack = None

    def get_object_at(self, offset):

        if offset in self._offset_cache:

            return self._offset_cache[offset]

        assert offset >= self._header_size

        pack_reader = SwiftPackReader(self.scon, self._filename,

                                      self.pack_length)

        pack_reader.seek(offset)

        unpacked, _ = unpack_object(pack_reader.read)

        return (unpacked.pack_type_num, unpacked._obj())

    def get_stored_checksum(self):

        pack_reader = SwiftPackReader(self.scon, self._filename,

                                      self.pack_length)

        return pack_reader.read_checksum()

    def close(self):

        pass

class SwiftPack(Pack):

    """A Git pack object.

    Same implementation as pack.Pack except that _idx_load and

    _data_load are bounded to Swift version of load_pack_index and

    PackData.

    """

    def __init__(self, *args, **kwargs):

        self.scon = kwargs['scon']

        del kwargs['scon']

        super(SwiftPack, self).__init__(*args, **kwargs)

        self._pack_info_path = self._basename + '.info'

        self._pack_info = None

        self._pack_info_load = lambda: load_pack_info(self._pack_info_path,

                                                      self.scon)

        self._idx_load = lambda: swift_load_pack_index(self.scon,

                                                       self._idx_path)

        self._data_load = lambda: SwiftPackData(self.scon, self._data_path)

    @property

    def pack_info(self):

        """The pack data object being used."""

        if self._pack_info is None:

            self._pack_info = self._pack_info_load()

        return self._pack_info

class SwiftObjectStore(PackBasedObjectStore):

    """A Swift Object Store

    Allow to manage a bare Git repository from Openstack Swift.

    This object store only supports pack files and not loose objects.

    """

    def __init__(self, scon):

        """Open a Swift object store.

        :param scon: A `SwiftConnector` instance

        """

        super(SwiftObjectStore, self).__init__()

        self.scon = scon

        self.root = self.scon.root

        self.pack_dir = posixpath.join(OBJECTDIR, PACKDIR)

        self._alternates = None

    @property

    def packs(self):

        """List with pack objects."""

        if not self._pack_cache:

            self._update_pack_cache()

        return self._pack_cache.values()

    def _update_pack_cache(self):

        for pack in self._load_packs():

            self._pack_cache[pack._basename] = pack

    def _iter_loose_objects(self):

        """Loose objects are not supported by this repository

        """

        return []

    def iter_shas(self, finder):

        """An iterator over pack's ObjectStore.

        :return: a `ObjectStoreIterator` or `GreenThreadsObjectStoreIterator`

                 instance if gevent is enabled

        """

        shas = iter(finder.next, None)

        return PackInfoObjectStoreIterator(

            self, shas, finder, self.scon.concurrency)

    def find_missing_objects(self, *args, **kwargs):

        kwargs['concurrency'] = self.scon.concurrency

        return PackInfoMissingObjectFinder(self, *args, **kwargs)

    def _load_packs(self):

        """Load all packs from Swift

        :return: a list of `SwiftPack` instances

        """

        objects = self.scon.get_container_objects()

        pack_files = [o['name'].replace(".pack", "")

                      for o in objects if o['name'].endswith(".pack")]

        return [SwiftPack(pack, scon=self.scon) for pack in pack_files]

    def pack_info_get(self, sha):

        for pack in self.packs:

            if sha in pack:

                return pack.pack_info[sha]

    def _collect_ancestors(self, heads, common=set()):

        def _find_parents(commit):

            for pack in self.packs:

                if commit in pack:

                    try:

                        parents = pack.pack_info[commit][1]

                    except KeyError:

                        # Seems to have no parents

                        return []

                    return parents

        bases = set()

        commits = set()

        queue = []

        queue.extend(heads)

        while queue:

            e = queue.pop(0)

            if e in common:

                bases.add(e)

            elif e not in commits:

                commits.add(e)

                parents = _find_parents(e)

                queue.extend(parents)

        return (commits, bases)

    def add_pack(self):

        """Add a new pack to this object store.

        :return: Fileobject to write to and a commit function to

            call when the pack is finished.

        """

        f = BytesIO()

        def commit():

            f.seek(0)

            pack = PackData(file=f, filename="")

            entries = pack.sorted_entries()

            if len(entries):

                basename = posixpath.join(self.pack_dir,

                                          "pack-%s" %

                                          iter_sha1(entry[0] for

                                                    entry in entries))

                index = BytesIO()

                write_pack_index_v2(index, entries, pack.get_stored_checksum())

                self.scon.put_object(basename + ".pack", f)

                f.close()

                self.scon.put_object(basename + ".idx", index)

                index.close()

                final_pack = SwiftPack(basename, scon=self.scon)

                final_pack.check_length_and_checksum()

                self._add_known_pack(basename, final_pack)

                return final_pack

            else:

                return None

        def abort():

            pass

        return f, commit, abort

    def add_object(self, obj):

        self.add_objects([(obj, None), ])

    def _pack_cache_stale(self):

        return False

    def _get_loose_object(self, sha):

        return None

    def add_thin_pack(self, read_all, read_some):

        """Read a thin pack

        Read it from a stream and complete it in a temporary file.

        Then the pack and the corresponding index file are uploaded to Swift.

        """

        fd, path = tempfile.mkstemp(prefix='tmp_pack_')

        f = os.fdopen(fd, 'w+b')

        try:

            indexer = PackIndexer(f, resolve_ext_ref=self.get_raw)

            copier = PackStreamCopier(read_all, read_some, f,

                                      delta_iter=indexer)

            copier.verify()

            return self._complete_thin_pack(f, path, copier, indexer)

        finally:

            f.close()

            os.unlink(path)

    def _complete_thin_pack(self, f, path, copier, indexer):

        entries = list(indexer)

        # Update the header with the new number of objects.

        f.seek(0)

        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Must flush before reading (http://bugs.python.org/issue3207)

        f.flush()

        # Rescan the rest of the pack, computing the SHA with the new header.

        new_sha = compute_file_sha(f, end_ofs=-20)

        # Must reposition before writing (http://bugs.python.org/issue3207)

        f.seek(0, os.SEEK_CUR)

        # Complete the pack.

        for ext_sha in indexer.ext_refs():

            assert len(ext_sha) == 20

            type_num, data = self.get_raw(ext_sha)

            offset = f.tell()

            crc32 = write_pack_object(f, type_num, data, sha=new_sha)

            entries.append((ext_sha, offset, crc32))

        pack_sha = new_sha.digest()

        f.write(pack_sha)

        f.flush()

        # Move the pack in.

        entries.sort()

        pack_base_name = posixpath.join(

            self.pack_dir,

            'pack-' + iter_sha1(e[0] for e in entries).decode(sys.getfilesystemencoding()))

        self.scon.put_object(pack_base_name + '.pack', f)

        # Write the index.

        filename = pack_base_name + '.idx'

        index_file = BytesIO()

        write_pack_index_v2(index_file, entries, pack_sha)

        self.scon.put_object(filename, index_file)

        # Write pack info.

        f.seek(0)

        pack_data = PackData(filename="", file=f)

        index_file.seek(0)

        pack_index = load_pack_index_file('', index_file)

        serialized_pack_info = pack_info_create(pack_data, pack_index)

        f.close()

        index_file.close()

        pack_info_file = BytesIO(serialized_pack_info)

        filename = pack_base_name + '.info'

        self.scon.put_object(filename, pack_info_file)

        pack_info_file.close()

        # Add the pack to the store and return it.

        final_pack = SwiftPack(pack_base_name, scon=self.scon)

        final_pack.check_length_and_checksum()

        self._add_known_pack(pack_base_name, final_pack)

        return final_pack

class SwiftInfoRefsContainer(InfoRefsContainer):

    """Manage references in info/refs object.

    """

    def __init__(self, scon, store):

        self.scon = scon

        self.filename = 'info/refs'

        self.store = store

        f = self.scon.get_object(self.filename)

        if not f:

            f = BytesIO(b'')

        super(SwiftInfoRefsContainer, self).__init__(f)

    def _load_check_ref(self, name, old_ref):

        self._check_refname(name)

        f = self.scon.get_object(self.filename)

        if not f:

            return {}

        refs = read_info_refs(f)

        if old_ref is not None:

            if refs[name] != old_ref:

                return False

        return refs

    def _write_refs(self, refs):

        f = BytesIO()

        f.writelines(write_info_refs(refs, self.store))

        self.scon.put_object(self.filename, f)

    def set_if_equals(self, name, old_ref, new_ref):

        """Set a refname to new_ref only if it currently equals old_ref.

        """

        if name == 'HEAD':

            return True

        refs = self._load_check_ref(name, old_ref)

        if not isinstance(refs, dict):

            return False

        refs[name] = new_ref

        self._write_refs(refs)

        self._refs[name] = new_ref

        return True

    def remove_if_equals(self, name, old_ref):

        """Remove a refname only if it currently equals old_ref.

        """

        if name == 'HEAD':

            return True

        refs = self._load_check_ref(name, old_ref)

        if not isinstance(refs, dict):

            return False

        del refs[name]

        self._write_refs(refs)

        del self._refs[name]

        return True

    def allkeys(self):

        try:

            self._refs['HEAD'] = self._refs['refs/heads/master']

        except KeyError:

            pass

        return self._refs.keys()

class SwiftRepo(BaseRepo):

    def __init__(self, root, conf):

        """Init a Git bare Repository on top of a Swift container.

        References are managed in info/refs objects by

        `SwiftInfoRefsContainer`. The root attribute is the Swift

        container that contain the Git bare repository.

        :param root: The container which contains the bare repo

        :param conf: A ConfigParser object

        """

        self.root = root.lstrip('/')

        self.conf = conf

        self.scon = SwiftConnector(self.root, self.conf)

        objects = self.scon.get_container_objects()

        if not objects:

            raise Exception('There is not any GIT repo here : %s' % self.root)

        objects = [o['name'].split('/')[0] for o in objects]

        if OBJECTDIR not in objects:

            raise Exception('This repository (%s) is not bare.' % self.root)

        self.bare = True

        self._controldir = self.root

        object_store = SwiftObjectStore(self.scon)

        refs = SwiftInfoRefsContainer(self.scon, object_store)

        BaseRepo.__init__(self, object_store, refs)

    def _determine_file_mode(self):

        """Probe the file-system to determine whether permissions can be trusted.

        :return: True if permissions can be trusted, False otherwise.

        """

        return False

    def _put_named_file(self, filename, contents):

        """Put an object in a Swift container

        :param filename: the path to the object to put on Swift

        :param contents: the content as bytestring

        """

        f = BytesIO()

        f.write(contents)

        self.scon.put_object(filename, f)

        f.close()

    @classmethod

    def init_bare(cls, scon, conf):

        """Create a new bare repository.

        :param scon: a `SwiftConnector` instance

        :param conf: a ConfigParser object

        :return: a `SwiftRepo` instance

        """

        scon.create_root()

        for obj in [posixpath.join(OBJECTDIR, PACKDIR),

                    posixpath.join(INFODIR, 'refs')]:

            scon.put_object(obj, BytesIO(b''))

        ret = cls(scon.root, conf)

        ret._init_files(True)

        return ret

class SwiftSystemBackend(Backend):

    def __init__(self, logger, conf):

        self.conf = conf

        self.logger = logger

    def open_repository(self, path):

        self.logger.info('opening repository at %s', path)

        return SwiftRepo(path, self.conf)

def cmd_daemon(args):

    """Entry point for starting a TCP git server."""

    import optparse

    parser = optparse.OptionParser()

    parser.add_option("-l", "--listen_address", dest="listen_address",

                      default="127.0.0.1",

                      help="Binding IP address.")

    parser.add_option("-p", "--port", dest="port", type=int,

                      default=TCP_GIT_PORT,

                      help="Binding TCP port.")

    parser.add_option("-c", "--swift_config", dest="swift_config",

                      default="",

                      help="Path to the configuration file for Swift backend.")

    options, args = parser.parse_args(args)

    try:

        import gevent

        import geventhttpclient

    except ImportError:

        print("gevent and geventhttpclient libraries are mandatory "

              " for use the Swift backend.")

        sys.exit(1)

    import gevent.monkey

    gevent.monkey.patch_socket()

    from dulwich.contrib.swift import load_conf

    from dulwich import log_utils

    logger = log_utils.getLogger(__name__)

    conf = load_conf(options.swift_config)

    backend = SwiftSystemBackend(logger, conf)

    log_utils.default_logging_config()

    server = TCPGitServer(backend, options.listen_address,

                          port=options.port)

    server.serve_forever()

def cmd_init(args):

    import optparse

    parser = optparse.OptionParser()

    parser.add_option("-c", "--swift_config", dest="swift_config",

                      default="",

                      help="Path to the configuration file for Swift backend.")

    options, args = parser.parse_args(args)

    conf = load_conf(options.swift_config)

    if args == []:

        parser.error("missing repository name")

    repo = args[0]

    scon = SwiftConnector(repo, conf)

    SwiftRepo.init_bare(scon, conf)

def main(argv=sys.argv):

    commands = {

        "init": cmd_init,

        "daemon": cmd_daemon,

    }

    if len(sys.argv) < 2:

        print("Usage: %s <%s> [OPTIONS...]" % (sys.argv[0], "|".join(commands.keys())))

        sys.exit(1)

    cmd = sys.argv[1]

    if not cmd in commands:

        print("No such subcommand: %s" % cmd)

        sys.exit(1)

    commands[cmd](sys.argv[2:])

if __name__ == '__main__':

    main()