gvsig-scripting / org.gvsig.scripting / trunk / org.gvsig.scripting / org.gvsig.scripting.app / org.gvsig.scripting.app.mainplugin / src / main / resources-plugin / scripting / lib / dulwich / server.py @ 959
History | View | Annotate | Download (38.7 KB)
1 |
# server.py -- Implementation of the server side git protocols
|
---|---|
2 |
# Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk>
|
3 |
# Coprygith (C) 2011-2012 Jelmer Vernooij <jelmer@samba.org>
|
4 |
#
|
5 |
# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
|
6 |
# General Public License as public by the Free Software Foundation; version 2.0
|
7 |
# or (at your option) any later version. You can redistribute it and/or
|
8 |
# modify it under the terms of either of these two licenses.
|
9 |
#
|
10 |
# Unless required by applicable law or agreed to in writing, software
|
11 |
# distributed under the License is distributed on an "AS IS" BASIS,
|
12 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13 |
# See the License for the specific language governing permissions and
|
14 |
# limitations under the License.
|
15 |
#
|
16 |
# You should have received a copy of the licenses; if not, see
|
17 |
# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
|
18 |
# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
|
19 |
# License, Version 2.0.
|
20 |
#
|
21 |
|
22 |
"""Git smart network protocol server implementation.
|
23 |
|
24 |
For more detailed implementation on the network protocol, see the
|
25 |
Documentation/technical directory in the cgit distribution, and in particular:
|
26 |
|
27 |
* Documentation/technical/protocol-capabilities.txt
|
28 |
* Documentation/technical/pack-protocol.txt
|
29 |
|
30 |
Currently supported capabilities:
|
31 |
|
32 |
* include-tag
|
33 |
* thin-pack
|
34 |
* multi_ack_detailed
|
35 |
* multi_ack
|
36 |
* side-band-64k
|
37 |
* ofs-delta
|
38 |
* no-progress
|
39 |
* report-status
|
40 |
* delete-refs
|
41 |
* shallow
|
42 |
"""
|
43 |
|
44 |
import collections |
45 |
import os |
46 |
import socket |
47 |
import sys |
48 |
import zlib |
49 |
|
50 |
try:
|
51 |
import SocketServer |
52 |
except ImportError: |
53 |
import socketserver as SocketServer |
54 |
|
55 |
from dulwich.errors import ( |
56 |
ApplyDeltaError, |
57 |
ChecksumMismatch, |
58 |
GitProtocolError, |
59 |
NotGitRepository, |
60 |
UnexpectedCommandError, |
61 |
ObjectFormatException, |
62 |
) |
63 |
from dulwich import log_utils |
64 |
from dulwich.objects import ( |
65 |
Commit, |
66 |
valid_hexsha, |
67 |
) |
68 |
from dulwich.pack import ( |
69 |
write_pack_objects, |
70 |
) |
71 |
from dulwich.protocol import ( |
72 |
BufferedPktLineWriter, |
73 |
capability_agent, |
74 |
CAPABILITIES_REF, |
75 |
CAPABILITY_DELETE_REFS, |
76 |
CAPABILITY_INCLUDE_TAG, |
77 |
CAPABILITY_MULTI_ACK_DETAILED, |
78 |
CAPABILITY_MULTI_ACK, |
79 |
CAPABILITY_NO_DONE, |
80 |
CAPABILITY_NO_PROGRESS, |
81 |
CAPABILITY_OFS_DELTA, |
82 |
CAPABILITY_QUIET, |
83 |
CAPABILITY_REPORT_STATUS, |
84 |
CAPABILITY_SHALLOW, |
85 |
CAPABILITY_SIDE_BAND_64K, |
86 |
CAPABILITY_THIN_PACK, |
87 |
COMMAND_DEEPEN, |
88 |
COMMAND_DONE, |
89 |
COMMAND_HAVE, |
90 |
COMMAND_SHALLOW, |
91 |
COMMAND_UNSHALLOW, |
92 |
COMMAND_WANT, |
93 |
MULTI_ACK, |
94 |
MULTI_ACK_DETAILED, |
95 |
Protocol, |
96 |
ProtocolFile, |
97 |
ReceivableProtocol, |
98 |
SIDE_BAND_CHANNEL_DATA, |
99 |
SIDE_BAND_CHANNEL_PROGRESS, |
100 |
SIDE_BAND_CHANNEL_FATAL, |
101 |
SINGLE_ACK, |
102 |
TCP_GIT_PORT, |
103 |
ZERO_SHA, |
104 |
ack_type, |
105 |
extract_capabilities, |
106 |
extract_want_line_capabilities, |
107 |
) |
108 |
from dulwich.refs import ( |
109 |
ANNOTATED_TAG_SUFFIX, |
110 |
write_info_refs, |
111 |
) |
112 |
from dulwich.repo import ( |
113 |
Repo, |
114 |
) |
115 |
|
116 |
|
117 |
logger = log_utils.getLogger(__name__) |
118 |
|
119 |
|
120 |
class Backend(object): |
121 |
"""A backend for the Git smart server implementation."""
|
122 |
|
123 |
def open_repository(self, path): |
124 |
"""Open the repository at a path.
|
125 |
|
126 |
:param path: Path to the repository
|
127 |
:raise NotGitRepository: no git repository was found at path
|
128 |
:return: Instance of BackendRepo
|
129 |
"""
|
130 |
raise NotImplementedError(self.open_repository) |
131 |
|
132 |
|
133 |
class BackendRepo(object): |
134 |
"""Repository abstraction used by the Git server.
|
135 |
|
136 |
The methods required here are a subset of those provided by
|
137 |
dulwich.repo.Repo.
|
138 |
"""
|
139 |
|
140 |
object_store = None
|
141 |
refs = None
|
142 |
|
143 |
def get_refs(self): |
144 |
"""
|
145 |
Get all the refs in the repository
|
146 |
|
147 |
:return: dict of name -> sha
|
148 |
"""
|
149 |
raise NotImplementedError |
150 |
|
151 |
def get_peeled(self, name): |
152 |
"""Return the cached peeled value of a ref, if available.
|
153 |
|
154 |
:param name: Name of the ref to peel
|
155 |
:return: The peeled value of the ref. If the ref is known not point to
|
156 |
a tag, this will be the SHA the ref refers to. If no cached
|
157 |
information about a tag is available, this method may return None,
|
158 |
but it should attempt to peel the tag if possible.
|
159 |
"""
|
160 |
return None |
161 |
|
162 |
def fetch_objects(self, determine_wants, graph_walker, progress, |
163 |
get_tagged=None):
|
164 |
"""
|
165 |
Yield the objects required for a list of commits.
|
166 |
|
167 |
:param progress: is a callback to send progress messages to the client
|
168 |
:param get_tagged: Function that returns a dict of pointed-to sha -> tag
|
169 |
sha for including tags.
|
170 |
"""
|
171 |
raise NotImplementedError |
172 |
|
173 |
|
174 |
class DictBackend(Backend): |
175 |
"""Trivial backend that looks up Git repositories in a dictionary."""
|
176 |
|
177 |
def __init__(self, repos): |
178 |
self.repos = repos
|
179 |
|
180 |
def open_repository(self, path): |
181 |
logger.debug('Opening repository at %s', path)
|
182 |
try:
|
183 |
return self.repos[path] |
184 |
except KeyError: |
185 |
raise NotGitRepository(
|
186 |
"No git repository was found at %(path)s" % dict(path=path) |
187 |
) |
188 |
|
189 |
|
190 |
class FileSystemBackend(Backend): |
191 |
"""Simple backend that looks up Git repositories in the local file system."""
|
192 |
|
193 |
def __init__(self, root=os.sep): |
194 |
super(FileSystemBackend, self).__init__() |
195 |
self.root = (os.path.abspath(root) + os.sep).replace(os.sep * 2, os.sep) |
196 |
|
197 |
def open_repository(self, path): |
198 |
logger.debug('opening repository at %s', path)
|
199 |
abspath = os.path.abspath(os.path.join(self.root, path)) + os.sep
|
200 |
normcase_abspath = os.path.normcase(abspath) |
201 |
normcase_root = os.path.normcase(self.root)
|
202 |
if not normcase_abspath.startswith(normcase_root): |
203 |
raise NotGitRepository("Path %r not inside root %r" % (path, self.root)) |
204 |
return Repo(abspath)
|
205 |
|
206 |
|
207 |
class Handler(object): |
208 |
"""Smart protocol command handler base class."""
|
209 |
|
210 |
def __init__(self, backend, proto, http_req=None): |
211 |
self.backend = backend
|
212 |
self.proto = proto
|
213 |
self.http_req = http_req
|
214 |
|
215 |
def handle(self): |
216 |
raise NotImplementedError(self.handle) |
217 |
|
218 |
|
219 |
class PackHandler(Handler): |
220 |
"""Protocol handler for packs."""
|
221 |
|
222 |
def __init__(self, backend, proto, http_req=None): |
223 |
super(PackHandler, self).__init__(backend, proto, http_req) |
224 |
self._client_capabilities = None |
225 |
# Flags needed for the no-done capability
|
226 |
self._done_received = False |
227 |
|
228 |
@classmethod
|
229 |
def capability_line(cls): |
230 |
return b"".join([b" " + c for c in cls.capabilities()]) |
231 |
|
232 |
@classmethod
|
233 |
def capabilities(cls): |
234 |
raise NotImplementedError(cls.capabilities) |
235 |
|
236 |
@classmethod
|
237 |
def innocuous_capabilities(cls): |
238 |
return (CAPABILITY_INCLUDE_TAG, CAPABILITY_THIN_PACK,
|
239 |
CAPABILITY_NO_PROGRESS, CAPABILITY_OFS_DELTA, |
240 |
capability_agent()) |
241 |
|
242 |
@classmethod
|
243 |
def required_capabilities(cls): |
244 |
"""Return a list of capabilities that we require the client to have."""
|
245 |
return []
|
246 |
|
247 |
def set_client_capabilities(self, caps): |
248 |
allowable_caps = set(self.innocuous_capabilities()) |
249 |
allowable_caps.update(self.capabilities())
|
250 |
for cap in caps: |
251 |
if cap not in allowable_caps: |
252 |
raise GitProtocolError('Client asked for capability %s that ' |
253 |
'was not advertised.' % cap)
|
254 |
for cap in self.required_capabilities(): |
255 |
if cap not in caps: |
256 |
raise GitProtocolError('Client does not support required ' |
257 |
'capability %s.' % cap)
|
258 |
self._client_capabilities = set(caps) |
259 |
logger.info('Client capabilities: %s', caps)
|
260 |
|
261 |
def has_capability(self, cap): |
262 |
if self._client_capabilities is None: |
263 |
raise GitProtocolError('Server attempted to access capability %s ' |
264 |
'before asking client' % cap)
|
265 |
return cap in self._client_capabilities |
266 |
|
267 |
def notify_done(self): |
268 |
self._done_received = True |
269 |
|
270 |
|
271 |
|
272 |
class UploadPackHandler(PackHandler): |
273 |
"""Protocol handler for uploading a pack to the client."""
|
274 |
|
275 |
def __init__(self, backend, args, proto, http_req=None, |
276 |
advertise_refs=False):
|
277 |
super(UploadPackHandler, self).__init__(backend, proto, |
278 |
http_req=http_req) |
279 |
self.repo = backend.open_repository(args[0]) |
280 |
self._graph_walker = None |
281 |
self.advertise_refs = advertise_refs
|
282 |
# A state variable for denoting that the have list is still
|
283 |
# being processed, and the client is not accepting any other
|
284 |
# data (such as side-band, see the progress method here).
|
285 |
self._processing_have_lines = False |
286 |
|
287 |
@classmethod
|
288 |
def capabilities(cls): |
289 |
return (CAPABILITY_MULTI_ACK_DETAILED, CAPABILITY_MULTI_ACK,
|
290 |
CAPABILITY_SIDE_BAND_64K, CAPABILITY_THIN_PACK, |
291 |
CAPABILITY_OFS_DELTA, CAPABILITY_NO_PROGRESS, |
292 |
CAPABILITY_INCLUDE_TAG, CAPABILITY_SHALLOW, CAPABILITY_NO_DONE) |
293 |
|
294 |
@classmethod
|
295 |
def required_capabilities(cls): |
296 |
return (CAPABILITY_SIDE_BAND_64K, CAPABILITY_THIN_PACK, CAPABILITY_OFS_DELTA)
|
297 |
|
298 |
def progress(self, message): |
299 |
if self.has_capability(CAPABILITY_NO_PROGRESS) or self._processing_have_lines: |
300 |
return
|
301 |
self.proto.write_sideband(SIDE_BAND_CHANNEL_PROGRESS, message)
|
302 |
|
303 |
def get_tagged(self, refs=None, repo=None): |
304 |
"""Get a dict of peeled values of tags to their original tag shas.
|
305 |
|
306 |
:param refs: dict of refname -> sha of possible tags; defaults to all of
|
307 |
the backend's refs.
|
308 |
:param repo: optional Repo instance for getting peeled refs; defaults to
|
309 |
the backend's repo, if available
|
310 |
:return: dict of peeled_sha -> tag_sha, where tag_sha is the sha of a
|
311 |
tag whose peeled value is peeled_sha.
|
312 |
"""
|
313 |
if not self.has_capability(CAPABILITY_INCLUDE_TAG): |
314 |
return {}
|
315 |
if refs is None: |
316 |
refs = self.repo.get_refs()
|
317 |
if repo is None: |
318 |
repo = getattr(self.repo, "repo", None) |
319 |
if repo is None: |
320 |
# Bail if we don't have a Repo available; this is ok since
|
321 |
# clients must be able to handle if the server doesn't include
|
322 |
# all relevant tags.
|
323 |
# TODO: fix behavior when missing
|
324 |
return {}
|
325 |
tagged = {} |
326 |
for name, sha in refs.items(): |
327 |
peeled_sha = repo.get_peeled(name) |
328 |
if peeled_sha != sha:
|
329 |
tagged[peeled_sha] = sha |
330 |
return tagged
|
331 |
|
332 |
def handle(self): |
333 |
write = lambda x: self.proto.write_sideband(SIDE_BAND_CHANNEL_DATA, x) |
334 |
|
335 |
graph_walker = ProtocolGraphWalker(self, self.repo.object_store, |
336 |
self.repo.get_peeled)
|
337 |
objects_iter = self.repo.fetch_objects(
|
338 |
graph_walker.determine_wants, graph_walker, self.progress,
|
339 |
get_tagged=self.get_tagged)
|
340 |
|
341 |
# Note the fact that client is only processing responses related
|
342 |
# to the have lines it sent, and any other data (including side-
|
343 |
# band) will be be considered a fatal error.
|
344 |
self._processing_have_lines = True |
345 |
|
346 |
# Did the process short-circuit (e.g. in a stateless RPC call)? Note
|
347 |
# that the client still expects a 0-object pack in most cases.
|
348 |
# Also, if it also happens that the object_iter is instantiated
|
349 |
# with a graph walker with an implementation that talks over the
|
350 |
# wire (which is this instance of this class) this will actually
|
351 |
# iterate through everything and write things out to the wire.
|
352 |
if len(objects_iter) == 0: |
353 |
return
|
354 |
|
355 |
# The provided haves are processed, and it is safe to send side-
|
356 |
# band data now.
|
357 |
self._processing_have_lines = False |
358 |
|
359 |
if not graph_walker.handle_done( |
360 |
not self.has_capability(CAPABILITY_NO_DONE), self._done_received): |
361 |
return
|
362 |
|
363 |
self.progress(b"dul-daemon says what\n") |
364 |
self.progress(("counting objects: %d, done.\n" % len(objects_iter)).encode('ascii')) |
365 |
write_pack_objects(ProtocolFile(None, write), objects_iter)
|
366 |
self.progress(b"how was that, then?\n") |
367 |
# we are done
|
368 |
self.proto.write_pkt_line(None) |
369 |
|
370 |
|
371 |
def _split_proto_line(line, allowed): |
372 |
"""Split a line read from the wire.
|
373 |
|
374 |
:param line: The line read from the wire.
|
375 |
:param allowed: An iterable of command names that should be allowed.
|
376 |
Command names not listed below as possible return values will be
|
377 |
ignored. If None, any commands from the possible return values are
|
378 |
allowed.
|
379 |
:return: a tuple having one of the following forms:
|
380 |
('want', obj_id)
|
381 |
('have', obj_id)
|
382 |
('done', None)
|
383 |
(None, None) (for a flush-pkt)
|
384 |
|
385 |
:raise UnexpectedCommandError: if the line cannot be parsed into one of the
|
386 |
allowed return values.
|
387 |
"""
|
388 |
if not line: |
389 |
fields = [None]
|
390 |
else:
|
391 |
fields = line.rstrip(b'\n').split(b' ', 1) |
392 |
command = fields[0]
|
393 |
if allowed is not None and command not in allowed: |
394 |
raise UnexpectedCommandError(command)
|
395 |
if len(fields) == 1 and command in (COMMAND_DONE, None): |
396 |
return (command, None) |
397 |
elif len(fields) == 2: |
398 |
if command in (COMMAND_WANT, COMMAND_HAVE, COMMAND_SHALLOW, |
399 |
COMMAND_UNSHALLOW): |
400 |
if not valid_hexsha(fields[1]): |
401 |
raise GitProtocolError("Invalid sha") |
402 |
return tuple(fields) |
403 |
elif command == COMMAND_DEEPEN:
|
404 |
return command, int(fields[1]) |
405 |
raise GitProtocolError('Received invalid line from client: %r' % line) |
406 |
|
407 |
|
408 |
def _find_shallow(store, heads, depth): |
409 |
"""Find shallow commits according to a given depth.
|
410 |
|
411 |
:param store: An ObjectStore for looking up objects.
|
412 |
:param heads: Iterable of head SHAs to start walking from.
|
413 |
:param depth: The depth of ancestors to include. A depth of one includes
|
414 |
only the heads themselves.
|
415 |
:return: A tuple of (shallow, not_shallow), sets of SHAs that should be
|
416 |
considered shallow and unshallow according to the arguments. Note that
|
417 |
these sets may overlap if a commit is reachable along multiple paths.
|
418 |
"""
|
419 |
parents = {} |
420 |
def get_parents(sha): |
421 |
result = parents.get(sha, None)
|
422 |
if not result: |
423 |
result = store[sha].parents |
424 |
parents[sha] = result |
425 |
return result
|
426 |
|
427 |
todo = [] # stack of (sha, depth)
|
428 |
for head_sha in heads: |
429 |
obj = store.peel_sha(head_sha) |
430 |
if isinstance(obj, Commit): |
431 |
todo.append((obj.id, 1))
|
432 |
|
433 |
not_shallow = set()
|
434 |
shallow = set()
|
435 |
while todo:
|
436 |
sha, cur_depth = todo.pop() |
437 |
if cur_depth < depth:
|
438 |
not_shallow.add(sha) |
439 |
new_depth = cur_depth + 1
|
440 |
todo.extend((p, new_depth) for p in get_parents(sha)) |
441 |
else:
|
442 |
shallow.add(sha) |
443 |
|
444 |
return shallow, not_shallow
|
445 |
|
446 |
|
447 |
def _want_satisfied(store, haves, want, earliest): |
448 |
o = store[want] |
449 |
pending = collections.deque([o]) |
450 |
while pending:
|
451 |
commit = pending.popleft() |
452 |
if commit.id in haves: |
453 |
return True |
454 |
if commit.type_name != b"commit": |
455 |
# non-commit wants are assumed to be satisfied
|
456 |
continue
|
457 |
for parent in commit.parents: |
458 |
parent_obj = store[parent] |
459 |
# TODO: handle parents with later commit times than children
|
460 |
if parent_obj.commit_time >= earliest:
|
461 |
pending.append(parent_obj) |
462 |
return False |
463 |
|
464 |
|
465 |
def _all_wants_satisfied(store, haves, wants): |
466 |
"""Check whether all the current wants are satisfied by a set of haves.
|
467 |
|
468 |
:param store: Object store to retrieve objects from
|
469 |
:param haves: A set of commits we know the client has.
|
470 |
:param wants: A set of commits the client wants
|
471 |
:note: Wants are specified with set_wants rather than passed in since
|
472 |
in the current interface they are determined outside this class.
|
473 |
"""
|
474 |
haves = set(haves)
|
475 |
if haves:
|
476 |
earliest = min([store[h].commit_time for h in haves]) |
477 |
else:
|
478 |
earliest = 0
|
479 |
for want in wants: |
480 |
if not _want_satisfied(store, haves, want, earliest): |
481 |
return False |
482 |
|
483 |
return True |
484 |
|
485 |
|
486 |
class ProtocolGraphWalker(object): |
487 |
"""A graph walker that knows the git protocol.
|
488 |
|
489 |
As a graph walker, this class implements ack(), next(), and reset(). It
|
490 |
also contains some base methods for interacting with the wire and walking
|
491 |
the commit tree.
|
492 |
|
493 |
The work of determining which acks to send is passed on to the
|
494 |
implementation instance stored in _impl. The reason for this is that we do
|
495 |
not know at object creation time what ack level the protocol requires. A
|
496 |
call to set_ack_level() is required to set up the implementation, before any
|
497 |
calls to next() or ack() are made.
|
498 |
"""
|
499 |
def __init__(self, handler, object_store, get_peeled): |
500 |
self.handler = handler
|
501 |
self.store = object_store
|
502 |
self.get_peeled = get_peeled
|
503 |
self.proto = handler.proto
|
504 |
self.http_req = handler.http_req
|
505 |
self.advertise_refs = handler.advertise_refs
|
506 |
self._wants = []
|
507 |
self.shallow = set() |
508 |
self.client_shallow = set() |
509 |
self.unshallow = set() |
510 |
self._cached = False |
511 |
self._cache = []
|
512 |
self._cache_index = 0 |
513 |
self._impl = None |
514 |
|
515 |
def determine_wants(self, heads): |
516 |
"""Determine the wants for a set of heads.
|
517 |
|
518 |
The given heads are advertised to the client, who then specifies which
|
519 |
refs he wants using 'want' lines. This portion of the protocol is the
|
520 |
same regardless of ack type, and in fact is used to set the ack type of
|
521 |
the ProtocolGraphWalker.
|
522 |
|
523 |
If the client has the 'shallow' capability, this method also reads and
|
524 |
responds to the 'shallow' and 'deepen' lines from the client. These are
|
525 |
not part of the wants per se, but they set up necessary state for
|
526 |
walking the graph. Additionally, later code depends on this method
|
527 |
consuming everything up to the first 'have' line.
|
528 |
|
529 |
:param heads: a dict of refname->SHA1 to advertise
|
530 |
:return: a list of SHA1s requested by the client
|
531 |
"""
|
532 |
values = set(heads.values())
|
533 |
if self.advertise_refs or not self.http_req: |
534 |
for i, (ref, sha) in enumerate(sorted(heads.items())): |
535 |
line = sha + b' ' + ref
|
536 |
if not i: |
537 |
line += b'\x00' + self.handler.capability_line() |
538 |
self.proto.write_pkt_line(line + b'\n') |
539 |
peeled_sha = self.get_peeled(ref)
|
540 |
if peeled_sha != sha:
|
541 |
self.proto.write_pkt_line(
|
542 |
peeled_sha + b' ' + ref + ANNOTATED_TAG_SUFFIX + b'\n') |
543 |
|
544 |
# i'm done..
|
545 |
self.proto.write_pkt_line(None) |
546 |
|
547 |
if self.advertise_refs: |
548 |
return []
|
549 |
|
550 |
# Now client will sending want want want commands
|
551 |
want = self.proto.read_pkt_line()
|
552 |
if not want: |
553 |
return []
|
554 |
line, caps = extract_want_line_capabilities(want) |
555 |
self.handler.set_client_capabilities(caps)
|
556 |
self.set_ack_type(ack_type(caps))
|
557 |
allowed = (COMMAND_WANT, COMMAND_SHALLOW, COMMAND_DEEPEN, None)
|
558 |
command, sha = _split_proto_line(line, allowed) |
559 |
|
560 |
want_revs = [] |
561 |
while command == COMMAND_WANT:
|
562 |
if sha not in values: |
563 |
raise GitProtocolError(
|
564 |
'Client wants invalid object %s' % sha)
|
565 |
want_revs.append(sha) |
566 |
command, sha = self.read_proto_line(allowed)
|
567 |
|
568 |
self.set_wants(want_revs)
|
569 |
if command in (COMMAND_SHALLOW, COMMAND_DEEPEN): |
570 |
self.unread_proto_line(command, sha)
|
571 |
self._handle_shallow_request(want_revs)
|
572 |
|
573 |
if self.http_req and self.proto.eof(): |
574 |
# The client may close the socket at this point, expecting a
|
575 |
# flush-pkt from the server. We might be ready to send a packfile at
|
576 |
# this point, so we need to explicitly short-circuit in this case.
|
577 |
return []
|
578 |
|
579 |
return want_revs
|
580 |
|
581 |
def unread_proto_line(self, command, value): |
582 |
if isinstance(value, int): |
583 |
value = str(value).encode('ascii') |
584 |
self.proto.unread_pkt_line(command + b' ' + value) |
585 |
|
586 |
def ack(self, have_ref): |
587 |
if len(have_ref) != 40: |
588 |
raise ValueError("invalid sha %r" % have_ref) |
589 |
return self._impl.ack(have_ref) |
590 |
|
591 |
def reset(self): |
592 |
self._cached = True |
593 |
self._cache_index = 0 |
594 |
|
595 |
def next(self): |
596 |
if not self._cached: |
597 |
if not self._impl and self.http_req: |
598 |
return None |
599 |
return next(self._impl) |
600 |
self._cache_index += 1 |
601 |
if self._cache_index > len(self._cache): |
602 |
return None |
603 |
return self._cache[self._cache_index] |
604 |
|
605 |
__next__ = next
|
606 |
|
607 |
def read_proto_line(self, allowed): |
608 |
"""Read a line from the wire.
|
609 |
|
610 |
:param allowed: An iterable of command names that should be allowed.
|
611 |
:return: A tuple of (command, value); see _split_proto_line.
|
612 |
:raise UnexpectedCommandError: If an error occurred reading the line.
|
613 |
"""
|
614 |
return _split_proto_line(self.proto.read_pkt_line(), allowed) |
615 |
|
616 |
def _handle_shallow_request(self, wants): |
617 |
while True: |
618 |
command, val = self.read_proto_line((COMMAND_DEEPEN, COMMAND_SHALLOW))
|
619 |
if command == COMMAND_DEEPEN:
|
620 |
depth = val |
621 |
break
|
622 |
self.client_shallow.add(val)
|
623 |
self.read_proto_line((None,)) # consume client's flush-pkt |
624 |
|
625 |
shallow, not_shallow = _find_shallow(self.store, wants, depth)
|
626 |
|
627 |
# Update self.shallow instead of reassigning it since we passed a
|
628 |
# reference to it before this method was called.
|
629 |
self.shallow.update(shallow - not_shallow)
|
630 |
new_shallow = self.shallow - self.client_shallow |
631 |
unshallow = self.unshallow = not_shallow & self.client_shallow |
632 |
|
633 |
for sha in sorted(new_shallow): |
634 |
self.proto.write_pkt_line(COMMAND_SHALLOW + b' ' + sha) |
635 |
for sha in sorted(unshallow): |
636 |
self.proto.write_pkt_line(COMMAND_UNSHALLOW + b' ' + sha) |
637 |
|
638 |
self.proto.write_pkt_line(None) |
639 |
|
640 |
def notify_done(self): |
641 |
# relay the message down to the handler.
|
642 |
self.handler.notify_done()
|
643 |
|
644 |
def send_ack(self, sha, ack_type=b''): |
645 |
if ack_type:
|
646 |
ack_type = b' ' + ack_type
|
647 |
self.proto.write_pkt_line(b'ACK ' + sha + ack_type + b'\n') |
648 |
|
649 |
def send_nak(self): |
650 |
self.proto.write_pkt_line(b'NAK\n') |
651 |
|
652 |
def handle_done(self, done_required, done_received): |
653 |
# Delegate this to the implementation.
|
654 |
return self._impl.handle_done(done_required, done_received) |
655 |
|
656 |
def set_wants(self, wants): |
657 |
self._wants = wants
|
658 |
|
659 |
def all_wants_satisfied(self, haves): |
660 |
"""Check whether all the current wants are satisfied by a set of haves.
|
661 |
|
662 |
:param haves: A set of commits we know the client has.
|
663 |
:note: Wants are specified with set_wants rather than passed in since
|
664 |
in the current interface they are determined outside this class.
|
665 |
"""
|
666 |
return _all_wants_satisfied(self.store, haves, self._wants) |
667 |
|
668 |
def set_ack_type(self, ack_type): |
669 |
impl_classes = { |
670 |
MULTI_ACK: MultiAckGraphWalkerImpl, |
671 |
MULTI_ACK_DETAILED: MultiAckDetailedGraphWalkerImpl, |
672 |
SINGLE_ACK: SingleAckGraphWalkerImpl, |
673 |
} |
674 |
self._impl = impl_classes[ack_type](self) |
675 |
|
676 |
|
677 |
_GRAPH_WALKER_COMMANDS = (COMMAND_HAVE, COMMAND_DONE, None)
|
678 |
|
679 |
|
680 |
class SingleAckGraphWalkerImpl(object): |
681 |
"""Graph walker implementation that speaks the single-ack protocol."""
|
682 |
|
683 |
def __init__(self, walker): |
684 |
self.walker = walker
|
685 |
self._common = []
|
686 |
|
687 |
def ack(self, have_ref): |
688 |
if not self._common: |
689 |
self.walker.send_ack(have_ref)
|
690 |
self._common.append(have_ref)
|
691 |
|
692 |
def next(self): |
693 |
command, sha = self.walker.read_proto_line(_GRAPH_WALKER_COMMANDS)
|
694 |
if command in (None, COMMAND_DONE): |
695 |
# defer the handling of done
|
696 |
self.walker.notify_done()
|
697 |
return None |
698 |
elif command == COMMAND_HAVE:
|
699 |
return sha
|
700 |
|
701 |
__next__ = next
|
702 |
|
703 |
def handle_done(self, done_required, done_received): |
704 |
if not self._common: |
705 |
self.walker.send_nak()
|
706 |
|
707 |
if done_required and not done_received: |
708 |
# we are not done, especially when done is required; skip
|
709 |
# the pack for this request and especially do not handle
|
710 |
# the done.
|
711 |
return False |
712 |
|
713 |
if not done_received and not self._common: |
714 |
# Okay we are not actually done then since the walker picked
|
715 |
# up no haves. This is usually triggered when client attempts
|
716 |
# to pull from a source that has no common base_commit.
|
717 |
# See: test_server.MultiAckDetailedGraphWalkerImplTestCase.\
|
718 |
# test_multi_ack_stateless_nodone
|
719 |
return False |
720 |
|
721 |
return True |
722 |
|
723 |
|
724 |
class MultiAckGraphWalkerImpl(object): |
725 |
"""Graph walker implementation that speaks the multi-ack protocol."""
|
726 |
|
727 |
def __init__(self, walker): |
728 |
self.walker = walker
|
729 |
self._found_base = False |
730 |
self._common = []
|
731 |
|
732 |
def ack(self, have_ref): |
733 |
self._common.append(have_ref)
|
734 |
if not self._found_base: |
735 |
self.walker.send_ack(have_ref, b'continue') |
736 |
if self.walker.all_wants_satisfied(self._common): |
737 |
self._found_base = True |
738 |
# else we blind ack within next
|
739 |
|
740 |
def next(self): |
741 |
while True: |
742 |
command, sha = self.walker.read_proto_line(_GRAPH_WALKER_COMMANDS)
|
743 |
if command is None: |
744 |
self.walker.send_nak()
|
745 |
# in multi-ack mode, a flush-pkt indicates the client wants to
|
746 |
# flush but more have lines are still coming
|
747 |
continue
|
748 |
elif command == COMMAND_DONE:
|
749 |
self.walker.notify_done()
|
750 |
return None |
751 |
elif command == COMMAND_HAVE:
|
752 |
if self._found_base: |
753 |
# blind ack
|
754 |
self.walker.send_ack(sha, b'continue') |
755 |
return sha
|
756 |
|
757 |
__next__ = next
|
758 |
|
759 |
def handle_done(self, done_required, done_received): |
760 |
if done_required and not done_received: |
761 |
# we are not done, especially when done is required; skip
|
762 |
# the pack for this request and especially do not handle
|
763 |
# the done.
|
764 |
return False |
765 |
|
766 |
if not done_received and not self._common: |
767 |
# Okay we are not actually done then since the walker picked
|
768 |
# up no haves. This is usually triggered when client attempts
|
769 |
# to pull from a source that has no common base_commit.
|
770 |
# See: test_server.MultiAckDetailedGraphWalkerImplTestCase.\
|
771 |
# test_multi_ack_stateless_nodone
|
772 |
return False |
773 |
|
774 |
# don't nak unless no common commits were found, even if not
|
775 |
# everything is satisfied
|
776 |
if self._common: |
777 |
self.walker.send_ack(self._common[-1]) |
778 |
else:
|
779 |
self.walker.send_nak()
|
780 |
return True |
781 |
|
782 |
|
783 |
class MultiAckDetailedGraphWalkerImpl(object): |
784 |
"""Graph walker implementation speaking the multi-ack-detailed protocol."""
|
785 |
|
786 |
def __init__(self, walker): |
787 |
self.walker = walker
|
788 |
self._common = []
|
789 |
|
790 |
def ack(self, have_ref): |
791 |
# Should only be called iff have_ref is common
|
792 |
self._common.append(have_ref)
|
793 |
self.walker.send_ack(have_ref, b'common') |
794 |
|
795 |
def next(self): |
796 |
while True: |
797 |
command, sha = self.walker.read_proto_line(_GRAPH_WALKER_COMMANDS)
|
798 |
if command is None: |
799 |
if self.walker.all_wants_satisfied(self._common): |
800 |
self.walker.send_ack(self._common[-1], b'ready') |
801 |
self.walker.send_nak()
|
802 |
if self.walker.http_req: |
803 |
# The HTTP version of this request a flush-pkt always
|
804 |
# signifies an end of request, so we also return
|
805 |
# nothing here as if we are done (but not really, as
|
806 |
# it depends on whether no-done capability was
|
807 |
# specified and that's handled in handle_done which
|
808 |
# may or may not call post_nodone_check depending on
|
809 |
# that).
|
810 |
return None |
811 |
elif command == COMMAND_DONE:
|
812 |
# Let the walker know that we got a done.
|
813 |
self.walker.notify_done()
|
814 |
break
|
815 |
elif command == COMMAND_HAVE:
|
816 |
# return the sha and let the caller ACK it with the
|
817 |
# above ack method.
|
818 |
return sha
|
819 |
# don't nak unless no common commits were found, even if not
|
820 |
# everything is satisfied
|
821 |
|
822 |
__next__ = next
|
823 |
|
824 |
def handle_done(self, done_required, done_received): |
825 |
if done_required and not done_received: |
826 |
# we are not done, especially when done is required; skip
|
827 |
# the pack for this request and especially do not handle
|
828 |
# the done.
|
829 |
return False |
830 |
|
831 |
if not done_received and not self._common: |
832 |
# Okay we are not actually done then since the walker picked
|
833 |
# up no haves. This is usually triggered when client attempts
|
834 |
# to pull from a source that has no common base_commit.
|
835 |
# See: test_server.MultiAckDetailedGraphWalkerImplTestCase.\
|
836 |
# test_multi_ack_stateless_nodone
|
837 |
return False |
838 |
|
839 |
# don't nak unless no common commits were found, even if not
|
840 |
# everything is satisfied
|
841 |
if self._common: |
842 |
self.walker.send_ack(self._common[-1]) |
843 |
else:
|
844 |
self.walker.send_nak()
|
845 |
return True |
846 |
|
847 |
|
848 |
class ReceivePackHandler(PackHandler): |
849 |
"""Protocol handler for downloading a pack from the client."""
|
850 |
|
851 |
def __init__(self, backend, args, proto, http_req=None, |
852 |
advertise_refs=False):
|
853 |
super(ReceivePackHandler, self).__init__(backend, proto, |
854 |
http_req=http_req) |
855 |
self.repo = backend.open_repository(args[0]) |
856 |
self.advertise_refs = advertise_refs
|
857 |
|
858 |
@classmethod
|
859 |
def capabilities(cls): |
860 |
return (CAPABILITY_REPORT_STATUS, CAPABILITY_DELETE_REFS, CAPABILITY_QUIET,
|
861 |
CAPABILITY_OFS_DELTA, CAPABILITY_SIDE_BAND_64K, CAPABILITY_NO_DONE) |
862 |
|
863 |
def _apply_pack(self, refs): |
864 |
all_exceptions = (IOError, OSError, ChecksumMismatch, ApplyDeltaError, |
865 |
AssertionError, socket.error, zlib.error,
|
866 |
ObjectFormatException) |
867 |
status = [] |
868 |
will_send_pack = False
|
869 |
|
870 |
for command in refs: |
871 |
if command[1] != ZERO_SHA: |
872 |
will_send_pack = True
|
873 |
|
874 |
if will_send_pack:
|
875 |
# TODO: more informative error messages than just the exception string
|
876 |
try:
|
877 |
recv = getattr(self.proto, "recv", None) |
878 |
self.repo.object_store.add_thin_pack(self.proto.read, recv) |
879 |
status.append((b'unpack', b'ok')) |
880 |
except all_exceptions as e: |
881 |
status.append((b'unpack', str(e).replace('\n', ''))) |
882 |
# The pack may still have been moved in, but it may contain broken
|
883 |
# objects. We trust a later GC to clean it up.
|
884 |
else:
|
885 |
# The git protocol want to find a status entry related to unpack process
|
886 |
# even if no pack data has been sent.
|
887 |
status.append((b'unpack', b'ok')) |
888 |
|
889 |
for oldsha, sha, ref in refs: |
890 |
ref_status = b'ok'
|
891 |
try:
|
892 |
if sha == ZERO_SHA:
|
893 |
if not CAPABILITY_DELETE_REFS in self.capabilities(): |
894 |
raise GitProtocolError(
|
895 |
'Attempted to delete refs without delete-refs '
|
896 |
'capability.')
|
897 |
try:
|
898 |
self.repo.refs.remove_if_equals(ref, oldsha)
|
899 |
except all_exceptions:
|
900 |
ref_status = b'failed to delete'
|
901 |
else:
|
902 |
try:
|
903 |
self.repo.refs.set_if_equals(ref, oldsha, sha)
|
904 |
except all_exceptions:
|
905 |
ref_status = b'failed to write'
|
906 |
except KeyError as e: |
907 |
ref_status = b'bad ref'
|
908 |
status.append((ref, ref_status)) |
909 |
|
910 |
return status
|
911 |
|
912 |
def _report_status(self, status): |
913 |
if self.has_capability(CAPABILITY_SIDE_BAND_64K): |
914 |
writer = BufferedPktLineWriter( |
915 |
lambda d: self.proto.write_sideband(SIDE_BAND_CHANNEL_DATA, d)) |
916 |
write = writer.write |
917 |
|
918 |
def flush(): |
919 |
writer.flush() |
920 |
self.proto.write_pkt_line(None) |
921 |
else:
|
922 |
write = self.proto.write_pkt_line
|
923 |
flush = lambda: None |
924 |
|
925 |
for name, msg in status: |
926 |
if name == b'unpack': |
927 |
write(b'unpack ' + msg + b'\n') |
928 |
elif msg == b'ok': |
929 |
write(b'ok ' + name + b'\n') |
930 |
else:
|
931 |
write(b'ng ' + name + b' ' + msg + b'\n') |
932 |
write(None)
|
933 |
flush() |
934 |
|
935 |
def handle(self): |
936 |
if self.advertise_refs or not self.http_req: |
937 |
refs = sorted(self.repo.get_refs().items()) |
938 |
|
939 |
if not refs: |
940 |
refs = [(CAPABILITIES_REF, ZERO_SHA)] |
941 |
self.proto.write_pkt_line(
|
942 |
refs[0][1] + b' ' + refs[0][0] + b'\0' + |
943 |
self.capability_line() + b'\n') |
944 |
for i in range(1, len(refs)): |
945 |
ref = refs[i] |
946 |
self.proto.write_pkt_line(ref[1] + b' ' + ref[0] + b'\n') |
947 |
|
948 |
self.proto.write_pkt_line(None) |
949 |
if self.advertise_refs: |
950 |
return
|
951 |
|
952 |
client_refs = [] |
953 |
ref = self.proto.read_pkt_line()
|
954 |
|
955 |
# if ref is none then client doesnt want to send us anything..
|
956 |
if ref is None: |
957 |
return
|
958 |
|
959 |
ref, caps = extract_capabilities(ref) |
960 |
self.set_client_capabilities(caps)
|
961 |
|
962 |
# client will now send us a list of (oldsha, newsha, ref)
|
963 |
while ref:
|
964 |
client_refs.append(ref.split()) |
965 |
ref = self.proto.read_pkt_line()
|
966 |
|
967 |
# backend can now deal with this refs and read a pack using self.read
|
968 |
status = self._apply_pack(client_refs)
|
969 |
|
970 |
# when we have read all the pack from the client, send a status report
|
971 |
# if the client asked for it
|
972 |
if self.has_capability(CAPABILITY_REPORT_STATUS): |
973 |
self._report_status(status)
|
974 |
|
975 |
|
976 |
class UploadArchiveHandler(Handler): |
977 |
|
978 |
def __init__(self, backend, proto, http_req=None): |
979 |
super(UploadArchiveHandler, self).__init__(backend, proto, http_req) |
980 |
|
981 |
def handle(self): |
982 |
# TODO(jelmer)
|
983 |
raise NotImplementedError(self.handle) |
984 |
|
985 |
|
986 |
# Default handler classes for git services.
|
987 |
DEFAULT_HANDLERS = { |
988 |
b'git-upload-pack': UploadPackHandler,
|
989 |
b'git-receive-pack': ReceivePackHandler,
|
990 |
# b'git-upload-archive': UploadArchiveHandler,
|
991 |
} |
992 |
|
993 |
|
994 |
class TCPGitRequestHandler(SocketServer.StreamRequestHandler): |
995 |
|
996 |
def __init__(self, handlers, *args, **kwargs): |
997 |
self.handlers = handlers
|
998 |
SocketServer.StreamRequestHandler.__init__(self, *args, **kwargs)
|
999 |
|
1000 |
def handle(self): |
1001 |
proto = ReceivableProtocol(self.connection.recv, self.wfile.write) |
1002 |
command, args = proto.read_cmd() |
1003 |
logger.info('Handling %s request, args=%s', command, args)
|
1004 |
|
1005 |
cls = self.handlers.get(command, None) |
1006 |
if not callable(cls): |
1007 |
raise GitProtocolError('Invalid service %s' % command) |
1008 |
h = cls(self.server.backend, args, proto)
|
1009 |
h.handle() |
1010 |
|
1011 |
|
1012 |
class TCPGitServer(SocketServer.TCPServer): |
1013 |
|
1014 |
allow_reuse_address = True
|
1015 |
serve = SocketServer.TCPServer.serve_forever |
1016 |
|
1017 |
def _make_handler(self, *args, **kwargs): |
1018 |
return TCPGitRequestHandler(self.handlers, *args, **kwargs) |
1019 |
|
1020 |
def __init__(self, backend, listen_addr, port=TCP_GIT_PORT, handlers=None): |
1021 |
self.handlers = dict(DEFAULT_HANDLERS) |
1022 |
if handlers is not None: |
1023 |
self.handlers.update(handlers)
|
1024 |
self.backend = backend
|
1025 |
logger.info('Listening for TCP connections on %s:%d', listen_addr, port)
|
1026 |
SocketServer.TCPServer.__init__(self, (listen_addr, port),
|
1027 |
self._make_handler)
|
1028 |
|
1029 |
def verify_request(self, request, client_address): |
1030 |
logger.info('Handling request from %s', client_address)
|
1031 |
return True |
1032 |
|
1033 |
def handle_error(self, request, client_address): |
1034 |
logger.exception('Exception happened during processing of request '
|
1035 |
'from %s', client_address)
|
1036 |
|
1037 |
|
1038 |
def main(argv=sys.argv): |
1039 |
"""Entry point for starting a TCP git server."""
|
1040 |
import optparse |
1041 |
parser = optparse.OptionParser() |
1042 |
parser.add_option("-l", "--listen_address", dest="listen_address", |
1043 |
default="localhost",
|
1044 |
help="Binding IP address.")
|
1045 |
parser.add_option("-p", "--port", dest="port", type=int, |
1046 |
default=TCP_GIT_PORT, |
1047 |
help="Binding TCP port.")
|
1048 |
options, args = parser.parse_args(argv) |
1049 |
|
1050 |
log_utils.default_logging_config() |
1051 |
if len(args) > 1: |
1052 |
gitdir = args[1]
|
1053 |
else:
|
1054 |
gitdir = '.'
|
1055 |
from dulwich import porcelain |
1056 |
porcelain.daemon(gitdir, address=options.listen_address, |
1057 |
port=options.port) |
1058 |
|
1059 |
|
1060 |
def serve_command(handler_cls, argv=sys.argv, backend=None, inf=sys.stdin, |
1061 |
outf=sys.stdout): |
1062 |
"""Serve a single command.
|
1063 |
|
1064 |
This is mostly useful for the implementation of commands used by e.g. git+ssh.
|
1065 |
|
1066 |
:param handler_cls: `Handler` class to use for the request
|
1067 |
:param argv: execv-style command-line arguments. Defaults to sys.argv.
|
1068 |
:param backend: `Backend` to use
|
1069 |
:param inf: File-like object to read from, defaults to standard input.
|
1070 |
:param outf: File-like object to write to, defaults to standard output.
|
1071 |
:return: Exit code for use with sys.exit. 0 on success, 1 on failure.
|
1072 |
"""
|
1073 |
if backend is None: |
1074 |
backend = FileSystemBackend() |
1075 |
def send_fn(data): |
1076 |
outf.write(data) |
1077 |
outf.flush() |
1078 |
proto = Protocol(inf.read, send_fn) |
1079 |
handler = handler_cls(backend, argv[1:], proto)
|
1080 |
# FIXME: Catch exceptions and write a single-line summary to outf.
|
1081 |
handler.handle() |
1082 |
return 0 |
1083 |
|
1084 |
|
1085 |
def generate_info_refs(repo): |
1086 |
"""Generate an info refs file."""
|
1087 |
refs = repo.get_refs() |
1088 |
return write_info_refs(refs, repo.object_store)
|
1089 |
|
1090 |
|
1091 |
def generate_objects_info_packs(repo): |
1092 |
"""Generate an index for for packs."""
|
1093 |
for pack in repo.object_store.packs: |
1094 |
yield b'P ' + pack.data.filename.encode(sys.getfilesystemencoding()) + b'\n' |
1095 |
|
1096 |
|
1097 |
def update_server_info(repo): |
1098 |
"""Generate server info for dumb file access.
|
1099 |
|
1100 |
This generates info/refs and objects/info/packs,
|
1101 |
similar to "git update-server-info".
|
1102 |
"""
|
1103 |
repo._put_named_file(os.path.join('info', 'refs'), |
1104 |
b"".join(generate_info_refs(repo)))
|
1105 |
|
1106 |
repo._put_named_file(os.path.join('objects', 'info', 'packs'), |
1107 |
b"".join(generate_objects_info_packs(repo)))
|
1108 |
|
1109 |
|
1110 |
if __name__ == '__main__': |
1111 |
main() |