Code Repositories xandikos / 098941b
Create store package. Jelmer Vernooij 2 years ago
2 changed file(s) with 871 addition(s) and 871 deletion(s). Raw diff Collapse all Expand all
0 # Xandikos
1 # Copyright (C) 2016-2017 Jelmer Vernooij <jelmer@jelmer.uk>, et al.
2 #
3 # This program is free software; you can redistribute it and/or
4 # modify it under the terms of the GNU General Public License
5 # as published by the Free Software Foundation; version 3
6 # of the License or (at your option) any later version of
7 # the License.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software
16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
17 # MA 02110-1301, USA.
18
19 """Stores and store sets.
20
21 ETags (https://en.wikipedia.org/wiki/HTTP_ETag) used in this file
22 are always strong, and should be returned without wrapping quotes.
23 """
24
25 import logging
26 import mimetypes
27 import os
28 import shutil
29 import stat
30 import uuid
31
32 from dulwich.objects import Blob, Tree
33 import dulwich.repo
34
35 _DEFAULT_COMMITTER_IDENTITY = b'Xandikos <xandikos>'
36
37 STORE_TYPE_ADDRESSBOOK = 'addressbook'
38 STORE_TYPE_CALENDAR = 'calendar'
39 STORE_TYPE_PRINCIPAL = 'principal'
40 STORE_TYPE_OTHER = 'other'
41 VALID_STORE_TYPES = (
42 STORE_TYPE_ADDRESSBOOK,
43 STORE_TYPE_CALENDAR,
44 STORE_TYPE_PRINCIPAL,
45 STORE_TYPE_OTHER)
46
47 MIMETYPES = mimetypes.MimeTypes()
48 MIMETYPES.add_type('text/calendar', '.ics')
49 MIMETYPES.add_type('text/vcard', '.vcf')
50
51 DEFAULT_MIME_TYPE = 'application/octet-stream'
52 DEFAULT_ENCODING = 'utf-8'
53
54
55 logger = logging.getLogger(__name__)
56
57
58 class File(object):
59 """A file type handler."""
60
61 def __init__(self, content, content_type):
62 self.content = content
63 self.content_type = content_type
64
65 def validate(self):
66 """Verify that file contents are valid.
67
68 :raise InvalidFileContents: Raised if a file is not valid
69 """
70 pass
71
72 def describe(self, name):
73 """Describe the contents of this file.
74
75 Used in e.g. commit messages.
76 """
77 return name
78
79 def get_uid(self):
80 """Return UID.
81
82 :raise NotImplementedError: If UIDs aren't supported for this format
83 :raise KeyError: If there is no UID set on this file
84 :raise InvalidFileContents: If the file is misformatted
85 :return: UID
86 """
87 raise NotImplementedError(self.get_uid)
88
89 def describe_delta(self, name, previous):
90 """Describe the important difference between this and previous one.
91
92 :param name: File name
93 :param previous: Previous file to compare to.
94 :raise InvalidFileContents: If the file is misformatted
95 :return: List of strings describing change
96 """
97 assert name is not None
98 item_description = self.describe(name)
99 assert item_description is not None
100 if previous is None:
101 yield "Added " + item_description
102 else:
103 yield "Modified " + item_description
104
105
106 def open_by_content_type(content, content_type, extra_file_handlers):
107 """Open a file based on content type.
108
109 :param content: list of bytestrings with content
110 :param content_type: MIME type
111 :return: File instance
112 """
113 return extra_file_handlers.get(content_type.split(';')[0], File)(
114 content, content_type)
115
116
117 def open_by_extension(content, name, extra_file_handlers):
118 """Open a file based on the filename extension.
119
120 :param content: list of bytestrings with content
121 :param name: Name of file to open
122 :return: File instance
123 """
124 (mime_type, _) = MIMETYPES.guess_type(name)
125 if mime_type is None:
126 mime_type = DEFAULT_MIME_TYPE
127 return open_by_content_type(content, mime_type,
128 extra_file_handlers=extra_file_handlers)
129
130
131 class DuplicateUidError(Exception):
132 """UID already exists in store."""
133
134 def __init__(self, uid, existing_name, new_name):
135 self.uid = uid
136 self.existing_name = existing_name
137 self.new_name = new_name
138
139
140 class NoSuchItem(Exception):
141 """No such item."""
142
143 def __init__(self, name):
144 self.name = name
145
146
147 class InvalidETag(Exception):
148 """Unexpected value for etag."""
149
150 def __init__(self, name, expected_etag, got_etag):
151 self.name = name
152 self.expected_etag = expected_etag
153 self.got_etag = got_etag
154
155
156 class NotStoreError(Exception):
157 """Not a store."""
158
159 def __init__(self, path):
160 self.path = path
161
162
163 class InvalidFileContents(Exception):
164 """Invalid file contents."""
165
166 def __init__(self, content_type, data):
167 self.content_type = content_type
168 self.data = data
169
170
171 class Store(object):
172 """A object store."""
173
174 def __init__(self):
175 self.extra_file_handlers = {}
176
177 def load_extra_file_handler(self, file_handler):
178 self.extra_file_handlers[file_handler.content_type] = file_handler
179
180 def iter_with_etag(self):
181 """Iterate over all items in the store with etag.
182
183 :yield: (name, content_type, etag) tuples
184 """
185 raise NotImplementedError(self.iter_with_etag)
186
187 def get_file(self, name, content_type=None, etag=None):
188 """Get the contents of an object.
189
190 :return: A File object
191 """
192 if content_type is None:
193 return open_by_extension(
194 self._get_raw(name, etag), name,
195 extra_file_handlers=self.extra_file_handlers)
196 else:
197 return open_by_content_type(
198 self._get_raw(name, etag), content_type,
199 extra_file_handlers=self.extra_file_handlers)
200
201 def _get_raw(self, name, etag):
202 """Get the raw contents of an object.
203
204 :return: raw contents
205 """
206 raise NotImplementedError(self._get_raw)
207
208 def get_ctag(self):
209 """Return the ctag for this store."""
210 raise NotImplementedError(self.get_ctag)
211
212 def import_one(self, name, data, message=None, author=None,
213 replace_etag=None):
214 """Import a single object.
215
216 :param name: Name of the object
217 :param data: serialized object as list of bytes
218 :param message: Commit message
219 :param author: Optional author
220 :param replace_etag: Etag to replace
221 :raise NameExists: when the name already exists
222 :raise DuplicateUidError: when the uid already exists
223 :return: (name, etag)
224 """
225 raise NotImplementedError(self.import_one)
226
227 def delete_one(self, name, message=None, author=None, etag=None):
228 """Delete an item.
229
230 :param name: Filename to delete
231 :param author: Optional author
232 :param message: Commit message
233 :param etag: Optional mandatory etag of object to remove
234 :raise NoSuchItem: when the item doesn't exist
235 :raise InvalidETag: If the specified ETag doesn't match the current
236 """
237 raise NotImplementedError(self.delete_one)
238
239 def set_type(self, store_type):
240 """Set store type.
241
242 :param store_type: New store type (one of VALID_STORE_TYPES)
243 """
244 raise NotImplementedError(self.set_type)
245
246 def get_type(self):
247 """Get type of this store.
248
249 :return: one of VALID_STORE_TYPES
250 """
251 ret = STORE_TYPE_OTHER
252 for (name, content_type, etag) in self.iter_with_etag():
253 if content_type == 'text/calendar':
254 ret = STORE_TYPE_CALENDAR
255 elif content_type == 'text/vcard':
256 ret = STORE_TYPE_ADDRESSBOOK
257 return ret
258
259 def set_description(self, description):
260 """Set the extended description of this store.
261
262 :param description: String with description
263 """
264 raise NotImplementedError(self.set_description)
265
266 def get_description(self):
267 """Get the extended description of this store.
268 """
269 raise NotImplementedError(self.get_description)
270
271 def get_displayname(self):
272 """Get the display name of this store.
273 """
274 raise NotImplementedError(self.get_displayname)
275
276 def set_displayname(self):
277 """Set the display name of this store.
278 """
279 raise NotImplementedError(self.set_displayname)
280
281 def get_color(self):
282 """Get the color code for this store."""
283 raise NotImplementedError(self.get_color)
284
285 def set_color(self, color):
286 """Set the color code for this store."""
287 raise NotImplementedError(self.set_color)
288
289 def iter_changes(self, old_ctag, new_ctag):
290 """Get changes between two versions of this store.
291
292 :param old_ctag: Old ctag (None for empty Store)
293 :param new_ctag: New ctag
294 :return: Iterator over (name, content_type, old_etag, new_etag)
295 """
296 raise NotImplementedError(self.iter_changes)
297
298 def get_comment(self):
299 """Retrieve store comment.
300
301 :return: Comment
302 """
303 raise NotImplementedError(self.get_comment)
304
305 def set_comment(self, comment):
306 """Set comment.
307
308 :param comment: New comment to set
309 """
310 raise NotImplementedError(self.set_comment)
311
312 def destroy(self):
313 """Destroy this store."""
314 raise NotImplementedError(self.destroy)
315
316 def subdirectories(self):
317 """Returns subdirectories to probe for other stores.
318
319 :return: List of names
320 """
321 raise NotImplementedError(self.subdirectories)
322
323
324 class GitStore(Store):
325 """A Store backed by a Git Repository.
326 """
327
328 def __init__(self, repo, ref=b'refs/heads/master',
329 check_for_duplicate_uids=True):
330 super(GitStore, self).__init__()
331 self.ref = ref
332 self.repo = repo
333 # Maps uids to (sha, fname)
334 self._uid_to_fname = {}
335 self._check_for_duplicate_uids = check_for_duplicate_uids
336 # Set of blob ids that have already been scanned
337 self._fname_to_uid = {}
338
339 def __repr__(self):
340 return "%s(%r, ref=%r)" % (type(self).__name__, self.repo, self.ref)
341
342 @property
343 def path(self):
344 return self.repo.path
345
346 def _check_duplicate(self, uid, name, replace_etag):
347 if uid is not None and self._check_for_duplicate_uids:
348 self._scan_uids()
349 try:
350 (existing_name, _) = self._uid_to_fname[uid]
351 except KeyError:
352 pass
353 else:
354 if existing_name != name:
355 raise DuplicateUidError(uid, existing_name, name)
356
357 try:
358 etag = self._get_etag(name)
359 except KeyError:
360 etag = None
361 if replace_etag is not None and etag != replace_etag:
362 raise InvalidETag(name, etag, replace_etag)
363 return etag
364
365 def import_one(self, name, content_type, data, message=None, author=None,
366 replace_etag=None):
367 """Import a single object.
368
369 :param name: name of the object
370 :param content_type: Content type
371 :param data: serialized object as list of bytes
372 :param message: Commit message
373 :param author: Optional author
374 :param replace_etag: optional etag of object to replace
375 :raise InvalidETag: when the name already exists but with different
376 etag
377 :raise DuplicateUidError: when the uid already exists
378 :return: etag
379 """
380 if content_type is None:
381 fi = open_by_extension(data, name, self.extra_file_handlers)
382 else:
383 fi = open_by_content_type(data, content_type, self.extra_file_handlers)
384 if name is None:
385 name = str(uuid.uuid4())
386 extension = MIMETYPES.guess_extension(content_type)
387 if extension is not None:
388 name += extension
389 fi.validate()
390 try:
391 uid = fi.get_uid()
392 except (KeyError, NotImplementedError):
393 uid = None
394 self._check_duplicate(uid, name, replace_etag)
395 if message is None:
396 try:
397 old_fi = self.get_file(name, content_type, replace_etag)
398 except KeyError:
399 old_fi = None
400 message = '\n'.join(fi.describe_delta(name, old_fi))
401 etag = self._import_one(name, data, message, author=author)
402 return (name, etag.decode('ascii'))
403
404 def _get_raw(self, name, etag=None):
405 """Get the raw contents of an object.
406
407 :param name: Name of the item
408 :param etag: Optional etag
409 :return: raw contents as chunks
410 """
411 if etag is None:
412 etag = self._get_etag(name)
413 blob = self.repo.object_store[etag.encode('ascii')]
414 return blob.chunked
415
416 def _scan_uids(self):
417 removed = set(self._fname_to_uid.keys())
418 for (name, mode, sha) in self._iterblobs():
419 etag = sha.decode('ascii')
420 if name in removed:
421 removed.remove(name)
422 if (name in self._fname_to_uid and
423 self._fname_to_uid[name][0] == etag):
424 continue
425 blob = self.repo.object_store[sha]
426 fi = open_by_extension(blob.chunked, name,
427 self.extra_file_handlers)
428 try:
429 uid = fi.get_uid()
430 except KeyError:
431 logger.warning('No UID found in file %s', name)
432 uid = None
433 except InvalidFileContents:
434 logging.warning('Unable to parse file %s', name)
435 uid = None
436 except NotImplementedError:
437 # This file type doesn't support UIDs
438 uid = None
439 self._fname_to_uid[name] = (etag, uid)
440 if uid is not None:
441 self._uid_to_fname[uid] = (name, etag)
442 for name in removed:
443 (unused_etag, uid) = self._fname_to_uid[name]
444 if uid is not None:
445 del self._uid_to_fname[uid]
446 del self._fname_to_uid[name]
447
448 def _iterblobs(self, ctag=None):
449 raise NotImplementedError(self._iterblobs)
450
451 def iter_with_etag(self, ctag=None):
452 """Iterate over all items in the store with etag.
453
454 :param ctag: Ctag to iterate for
455 :yield: (name, content_type, etag) tuples
456 """
457 for (name, mode, sha) in self._iterblobs(ctag):
458 (mime_type, _) = MIMETYPES.guess_type(name)
459 if mime_type is None:
460 mime_type = DEFAULT_MIME_TYPE
461 yield (name, mime_type, sha.decode('ascii'))
462
463 @classmethod
464 def create(cls, path):
465 """Create a new store backed by a Git repository on disk.
466
467 :return: A `GitStore`
468 """
469 raise NotImplementedError(cls.create)
470
471 @classmethod
472 def open_from_path(cls, path):
473 """Open a GitStore from a path.
474
475 :param path: Path
476 :return: A `GitStore`
477 """
478 try:
479 return cls.open(dulwich.repo.Repo(path))
480 except dulwich.repo.NotGitRepository:
481 raise NotStoreError(path)
482
483 @classmethod
484 def open(cls, repo):
485 """Open a GitStore given a Repo object.
486
487 :param repo: A Dulwich `Repo`
488 :return: A `GitStore`
489 """
490 if repo.has_index():
491 return TreeGitStore(repo)
492 else:
493 return BareGitStore(repo)
494
495 def get_description(self):
496 """Get extended description.
497
498 :return: repository description as string
499 """
500 desc = self.repo.get_description()
501 if desc is not None:
502 desc = desc.decode(DEFAULT_ENCODING)
503 return desc
504
505 def set_description(self, description):
506 """Set extended description.
507
508 :param description: repository description as string
509 """
510 return self.repo.set_description(description.encode(DEFAULT_ENCODING))
511
512 def set_comment(self, comment):
513 """Set comment.
514
515 :param comment: Comment
516 """
517 config = self.repo.get_config()
518 config.set(b'xandikos', b'comment', comment.encode(DEFAULT_ENCODING))
519 config.write_to_path()
520
521 def get_comment(self):
522 """Get comment.
523
524 :return: Comment
525 """
526 config = self.repo.get_config()
527 try:
528 comment = config.get(b'xandikos', b'comment')
529 except KeyError:
530 return None
531 else:
532 return comment.decode(DEFAULT_ENCODING)
533
534 def get_color(self):
535 """Get color.
536
537 :return: A Color code, or None
538 """
539 config = self.repo.get_config()
540 try:
541 color = config.get(b'xandikos', b'color')
542 except KeyError:
543 return None
544 else:
545 return color.decode(DEFAULT_ENCODING)
546
547 def set_color(self, color):
548 """Set the color code for this store."""
549 config = self.repo.get_config()
550 # Strip leading # to work around
551 # https://github.com/jelmer/dulwich/issues/511
552 # TODO(jelmer): Drop when that bug gets fixed.
553 config.set(
554 b'xandikos', b'color',
555 color.lstrip('#').encode(DEFAULT_ENCODING) if color else b'')
556 config.write_to_path()
557
558 def get_displayname(self):
559 """Get display name.
560
561 :return: The display name, or None if not set
562 """
563 config = self.repo.get_config()
564 try:
565 displayname = config.get(b'xandikos', b'displayname')
566 except KeyError:
567 return None
568 else:
569 return displayname.decode(DEFAULT_ENCODING)
570
571 def set_displayname(self, displayname):
572 """Set the display name.
573
574 :param displayname: New display name
575 """
576 config = self.repo.get_config()
577 config.set(b'xandikos', b'displayname',
578 displayname.encode(DEFAULT_ENCODING))
579 config.write_to_path()
580
581 def set_type(self, store_type):
582 """Set store type.
583
584 :param store_type: New store type (one of VALID_STORE_TYPES)
585 """
586 config = self.repo.get_config()
587 config.set(b'xandikos', b'type', store_type.encode(DEFAULT_ENCODING))
588 config.write_to_path()
589
590 def get_type(self):
591 """Get store type.
592
593 This looks in git config first, then falls back to guessing.
594 """
595 config = self.repo.get_config()
596 try:
597 store_type = config.get(b'xandikos', b'type')
598 except KeyError:
599 return super(GitStore, self).get_type()
600 else:
601 store_type = store_type.decode(DEFAULT_ENCODING)
602 if store_type not in VALID_STORE_TYPES:
603 logging.warning(
604 'Invalid store type %s set for %r.',
605 store_type, self.repo)
606 return store_type
607
608 def iter_changes(self, old_ctag, new_ctag):
609 """Get changes between two versions of this store.
610
611 :param old_ctag: Old ctag (None for empty Store)
612 :param new_ctag: New ctag
613 :return: Iterator over (name, content_type, old_etag, new_etag)
614 """
615 if old_ctag is None:
616 t = Tree()
617 self.repo.object_store.add_object(t)
618 old_ctag = t.id.decode('ascii')
619 previous = {
620 name: (content_type, etag)
621 for (name, content_type, etag) in self.iter_with_etag(old_ctag)
622 }
623 for (name, new_content_type, new_etag) in (
624 self.iter_with_etag(new_ctag)):
625 try:
626 (old_content_type, old_etag) = previous[name]
627 except KeyError:
628 old_etag = None
629 else:
630 assert old_content_type == new_content_type
631 if old_etag != new_etag:
632 yield (name, new_content_type, old_etag, new_etag)
633 if old_etag is not None:
634 del previous[name]
635 for (name, (old_content_type, old_etag)) in previous.items():
636 yield (name, old_content_type, old_etag, None)
637
638 def destroy(self):
639 """Destroy this store."""
640 shutil.rmtree(self.path)
641
642
643 class BareGitStore(GitStore):
644 """A Store backed by a bare git repository."""
645
646 def _get_current_tree(self):
647 try:
648 ref_object = self.repo[self.ref]
649 except KeyError:
650 return Tree()
651 if isinstance(ref_object, Tree):
652 return ref_object
653 else:
654 return self.repo.object_store[ref_object.tree]
655
656 def _get_etag(self, name):
657 tree = self._get_current_tree()
658 name = name.encode(DEFAULT_ENCODING)
659 return tree[name][1].decode('ascii')
660
661 def get_ctag(self):
662 """Return the ctag for this store."""
663 return self._get_current_tree().id.decode('ascii')
664
665 def _iterblobs(self, ctag=None):
666 if ctag is None:
667 tree = self._get_current_tree()
668 else:
669 tree = self.repo.object_store[ctag.encode('ascii')]
670 for (name, mode, sha) in tree.iteritems():
671 name = name.decode(DEFAULT_ENCODING)
672 yield (name, mode, sha)
673
674 @classmethod
675 def create_memory(cls):
676 """Create a new store backed by a memory repository.
677
678 :return: A `GitStore`
679 """
680 return cls(dulwich.repo.MemoryRepo())
681
682 def _commit_tree(self, tree_id, message, author=None):
683 try:
684 committer = self.repo._get_user_identity()
685 except KeyError:
686 committer = _DEFAULT_COMMITTER_IDENTITY
687 return self.repo.do_commit(message=message, tree=tree_id, ref=self.ref,
688 committer=committer, author=author)
689
690 def _import_one(self, name, data, message, author=None):
691 """Import a single object.
692
693 :param name: Optional name of the object
694 :param data: serialized object as bytes
695 :param message: optional commit message
696 :param author: optional author
697 :return: etag
698 """
699 b = Blob()
700 b.chunked = data
701 tree = self._get_current_tree()
702 name_enc = name.encode(DEFAULT_ENCODING)
703 tree[name_enc] = (0o644 | stat.S_IFREG, b.id)
704 self.repo.object_store.add_objects([(tree, ''), (b, name_enc)])
705 self._commit_tree(tree.id, message.encode(DEFAULT_ENCODING),
706 author=author)
707 return b.id
708
709 def delete_one(self, name, message=None, author=None, etag=None):
710 """Delete an item.
711
712 :param name: Filename to delete
713 :param message; Commit message
714 :param author: Optional author to store
715 :param etag: Optional mandatory etag of object to remove
716 :raise NoSuchItem: when the item doesn't exist
717 :raise InvalidETag: If the specified ETag doesn't match the curren
718 """
719 tree = self._get_current_tree()
720 name_enc = name.encode(DEFAULT_ENCODING)
721 try:
722 current_sha = tree[name_enc][1]
723 except KeyError:
724 raise NoSuchItem(name)
725 if etag is not None and current_sha != etag.encode('ascii'):
726 raise InvalidETag(name, etag, current_sha.decode('ascii'))
727 del tree[name_enc]
728 self.repo.object_store.add_objects([(tree, '')])
729 if message is None:
730 fi = open_by_extension(
731 self.repo.object_store[current_sha].chunked, name,
732 self.extra_file_handlers)
733 message = "Delete " + fi.describe(name)
734 self._commit_tree(tree.id, message.encode(DEFAULT_ENCODING),
735 author=author)
736
737 @classmethod
738 def create(cls, path):
739 """Create a new store backed by a Git repository on disk.
740
741 :return: A `GitStore`
742 """
743 os.mkdir(path)
744 return cls(dulwich.repo.Repo.init_bare(path))
745
746 def subdirectories(self):
747 """Returns subdirectories to probe for other stores.
748
749 :return: List of names
750 """
751 # Or perhaps just return all subdirectories but filter out
752 # Git-owned ones?
753 return []
754
755
756 class TreeGitStore(GitStore):
757 """A Store that backs onto a treefull Git repository."""
758
759 @classmethod
760 def create(cls, path, bare=True):
761 """Create a new store backed by a Git repository on disk.
762
763 :return: A `GitStore`
764 """
765 os.mkdir(path)
766 return cls(dulwich.repo.Repo.init(path))
767
768 def _get_etag(self, name):
769 index = self.repo.open_index()
770 name = name.encode(DEFAULT_ENCODING)
771 return index[name].sha.decode('ascii')
772
773 def _commit_tree(self, message, author=None):
774 try:
775 committer = self.repo._get_user_identity()
776 except KeyError:
777 committer = _DEFAULT_COMMITTER_IDENTITY
778 return self.repo.do_commit(message=message, committer=committer,
779 author=author)
780
781 def _import_one(self, name, data, message, author=None):
782 """Import a single object.
783
784 :param name: name of the object
785 :param data: serialized object as list of bytes
786 :param message: Commit message
787 :param author: Optional author
788 :return: etag
789 """
790 p = os.path.join(self.repo.path, name)
791 with open(p, 'wb') as f:
792 f.writelines(data)
793 self.repo.stage(name)
794 etag = self.repo.open_index()[name.encode(DEFAULT_ENCODING)].sha
795 self._commit_tree(message.encode(DEFAULT_ENCODING), author=author)
796 return etag
797
798 def delete_one(self, name, message=None, author=None, etag=None):
799 """Delete an item.
800
801 :param name: Filename to delete
802 :param message: Commit message
803 :param author: Optional author
804 :param etag: Optional mandatory etag of object to remove
805 :raise NoSuchItem: when the item doesn't exist
806 :raise InvalidETag: If the specified ETag doesn't match the curren
807 """
808 p = os.path.join(self.repo.path, name)
809 try:
810 with open(p, 'rb') as f:
811 current_blob = Blob.from_string(f.read())
812 except IOError:
813 raise NoSuchItem(name)
814 if message is None:
815 fi = open_by_extension(current_blob.chunked, name,
816 self.extra_file_handlers)
817 message = 'Delete ' + fi.describe(name)
818 if etag is not None:
819 with open(p, 'rb') as f:
820 current_etag = current_blob.id
821 if etag.encode('ascii') != current_etag:
822 raise InvalidETag(name, etag, current_etag.decode('ascii'))
823 os.unlink(p)
824 self.repo.stage(name)
825 self._commit_tree(message.encode(DEFAULT_ENCODING), author=author)
826
827 def get_ctag(self):
828 """Return the ctag for this store."""
829 index = self.repo.open_index()
830 return index.commit(self.repo.object_store).decode('ascii')
831
832 def _iterblobs(self, ctag=None):
833 """Iterate over all items in the store with etag.
834
835 :yield: (name, etag) tuples
836 """
837 if ctag is not None:
838 tree = self.repo.object_store[ctag.encode('ascii')]
839 for (name, mode, sha) in tree.iteritems():
840 name = name.decode(DEFAULT_ENCODING)
841 yield (name, mode, sha)
842 else:
843 index = self.repo.open_index()
844 for (name, sha, mode) in index.iterblobs():
845 name = name.decode(DEFAULT_ENCODING)
846 yield (name, mode, sha)
847
848 def subdirectories(self):
849 """Returns subdirectories to probe for other stores.
850
851 :return: List of names
852 """
853 ret = []
854 for name in os.listdir(self.path):
855 if name == dulwich.repo.CONTROLDIR:
856 continue
857 p = os.path.join(self.path, name)
858 if os.path.isdir(p):
859 ret.append(name)
860 return ret
861
862
863 def open_store(location):
864 """Open store from a location string.
865
866 :param location: Location string to open
867 :return: A `Store`
868 """
869 # For now, just support opening git stores
870 return GitStore.open_from_path(location)
+0
-871
xandikos/store.py less more
0 # Xandikos
1 # Copyright (C) 2016-2017 Jelmer Vernooij <jelmer@jelmer.uk>, et al.
2 #
3 # This program is free software; you can redistribute it and/or
4 # modify it under the terms of the GNU General Public License
5 # as published by the Free Software Foundation; version 3
6 # of the License or (at your option) any later version of
7 # the License.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software
16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
17 # MA 02110-1301, USA.
18
19 """Stores and store sets.
20
21 ETags (https://en.wikipedia.org/wiki/HTTP_ETag) used in this file
22 are always strong, and should be returned without wrapping quotes.
23 """
24
25 import logging
26 import mimetypes
27 import os
28 import shutil
29 import stat
30 import uuid
31
32 from dulwich.objects import Blob, Tree
33 import dulwich.repo
34
35 _DEFAULT_COMMITTER_IDENTITY = b'Xandikos <xandikos>'
36
37 STORE_TYPE_ADDRESSBOOK = 'addressbook'
38 STORE_TYPE_CALENDAR = 'calendar'
39 STORE_TYPE_PRINCIPAL = 'principal'
40 STORE_TYPE_OTHER = 'other'
41 VALID_STORE_TYPES = (
42 STORE_TYPE_ADDRESSBOOK,
43 STORE_TYPE_CALENDAR,
44 STORE_TYPE_PRINCIPAL,
45 STORE_TYPE_OTHER)
46
47 MIMETYPES = mimetypes.MimeTypes()
48 MIMETYPES.add_type('text/calendar', '.ics')
49 MIMETYPES.add_type('text/vcard', '.vcf')
50
51 DEFAULT_MIME_TYPE = 'application/octet-stream'
52 DEFAULT_ENCODING = 'utf-8'
53
54
55 logger = logging.getLogger(__name__)
56
57
58 class File(object):
59 """A file type handler."""
60
61 def __init__(self, content, content_type):
62 self.content = content
63 self.content_type = content_type
64
65 def validate(self):
66 """Verify that file contents are valid.
67
68 :raise InvalidFileContents: Raised if a file is not valid
69 """
70 pass
71
72 def describe(self, name):
73 """Describe the contents of this file.
74
75 Used in e.g. commit messages.
76 """
77 return name
78
79 def get_uid(self):
80 """Return UID.
81
82 :raise NotImplementedError: If UIDs aren't supported for this format
83 :raise KeyError: If there is no UID set on this file
84 :raise InvalidFileContents: If the file is misformatted
85 :return: UID
86 """
87 raise NotImplementedError(self.get_uid)
88
89 def describe_delta(self, name, previous):
90 """Describe the important difference between this and previous one.
91
92 :param name: File name
93 :param previous: Previous file to compare to.
94 :raise InvalidFileContents: If the file is misformatted
95 :return: List of strings describing change
96 """
97 assert name is not None
98 item_description = self.describe(name)
99 assert item_description is not None
100 if previous is None:
101 yield "Added " + item_description
102 else:
103 yield "Modified " + item_description
104
105
106 def open_by_content_type(content, content_type, extra_file_handlers):
107 """Open a file based on content type.
108
109 :param content: list of bytestrings with content
110 :param content_type: MIME type
111 :return: File instance
112 """
113 return extra_file_handlers.get(content_type.split(';')[0], File)(
114 content, content_type)
115
116
117 def open_by_extension(content, name, extra_file_handlers):
118 """Open a file based on the filename extension.
119
120 :param content: list of bytestrings with content
121 :param name: Name of file to open
122 :return: File instance
123 """
124 (mime_type, _) = MIMETYPES.guess_type(name)
125 if mime_type is None:
126 mime_type = DEFAULT_MIME_TYPE
127 return open_by_content_type(content, mime_type,
128 extra_file_handlers=extra_file_handlers)
129
130
131 class DuplicateUidError(Exception):
132 """UID already exists in store."""
133
134 def __init__(self, uid, existing_name, new_name):
135 self.uid = uid
136 self.existing_name = existing_name
137 self.new_name = new_name
138
139
140 class NoSuchItem(Exception):
141 """No such item."""
142
143 def __init__(self, name):
144 self.name = name
145
146
147 class InvalidETag(Exception):
148 """Unexpected value for etag."""
149
150 def __init__(self, name, expected_etag, got_etag):
151 self.name = name
152 self.expected_etag = expected_etag
153 self.got_etag = got_etag
154
155
156 class NotStoreError(Exception):
157 """Not a store."""
158
159 def __init__(self, path):
160 self.path = path
161
162
163 class InvalidFileContents(Exception):
164 """Invalid file contents."""
165
166 def __init__(self, content_type, data):
167 self.content_type = content_type
168 self.data = data
169
170
171 class Store(object):
172 """A object store."""
173
174 def __init__(self):
175 self.extra_file_handlers = {}
176
177 def load_extra_file_handler(self, file_handler):
178 self.extra_file_handlers[file_handler.content_type] = file_handler
179
180 def iter_with_etag(self):
181 """Iterate over all items in the store with etag.
182
183 :yield: (name, content_type, etag) tuples
184 """
185 raise NotImplementedError(self.iter_with_etag)
186
187 def get_file(self, name, content_type=None, etag=None):
188 """Get the contents of an object.
189
190 :return: A File object
191 """
192 if content_type is None:
193 return open_by_extension(
194 self._get_raw(name, etag), name,
195 extra_file_handlers=self.extra_file_handlers)
196 else:
197 return open_by_content_type(
198 self._get_raw(name, etag), content_type,
199 extra_file_handlers=self.extra_file_handlers)
200
201 def _get_raw(self, name, etag):
202 """Get the raw contents of an object.
203
204 :return: raw contents
205 """
206 raise NotImplementedError(self._get_raw)
207
208 def get_ctag(self):
209 """Return the ctag for this store."""
210 raise NotImplementedError(self.get_ctag)
211
212 def import_one(self, name, data, message=None, author=None,
213 replace_etag=None):
214 """Import a single object.
215
216 :param name: Name of the object
217 :param data: serialized object as list of bytes
218 :param message: Commit message
219 :param author: Optional author
220 :param replace_etag: Etag to replace
221 :raise NameExists: when the name already exists
222 :raise DuplicateUidError: when the uid already exists
223 :return: (name, etag)
224 """
225 raise NotImplementedError(self.import_one)
226
227 def delete_one(self, name, message=None, author=None, etag=None):
228 """Delete an item.
229
230 :param name: Filename to delete
231 :param author: Optional author
232 :param message: Commit message
233 :param etag: Optional mandatory etag of object to remove
234 :raise NoSuchItem: when the item doesn't exist
235 :raise InvalidETag: If the specified ETag doesn't match the current
236 """
237 raise NotImplementedError(self.delete_one)
238
239 def set_type(self, store_type):
240 """Set store type.
241
242 :param store_type: New store type (one of VALID_STORE_TYPES)
243 """
244 raise NotImplementedError(self.set_type)
245
246 def get_type(self):
247 """Get type of this store.
248
249 :return: one of VALID_STORE_TYPES
250 """
251 ret = STORE_TYPE_OTHER
252 for (name, content_type, etag) in self.iter_with_etag():
253 if content_type == 'text/calendar':
254 ret = STORE_TYPE_CALENDAR
255 elif content_type == 'text/vcard':
256 ret = STORE_TYPE_ADDRESSBOOK
257 return ret
258
259 def set_description(self, description):
260 """Set the extended description of this store.
261
262 :param description: String with description
263 """
264 raise NotImplementedError(self.set_description)
265
266 def get_description(self):
267 """Get the extended description of this store.
268 """
269 raise NotImplementedError(self.get_description)
270
271 def get_displayname(self):
272 """Get the display name of this store.
273 """
274 raise NotImplementedError(self.get_displayname)
275
276 def set_displayname(self):
277 """Set the display name of this store.
278 """
279 raise NotImplementedError(self.set_displayname)
280
281 def get_color(self):
282 """Get the color code for this store."""
283 raise NotImplementedError(self.get_color)
284
285 def set_color(self, color):
286 """Set the color code for this store."""
287 raise NotImplementedError(self.set_color)
288
289 def iter_changes(self, old_ctag, new_ctag):
290 """Get changes between two versions of this store.
291
292 :param old_ctag: Old ctag (None for empty Store)
293 :param new_ctag: New ctag
294 :return: Iterator over (name, content_type, old_etag, new_etag)
295 """
296 raise NotImplementedError(self.iter_changes)
297
298 def get_comment(self):
299 """Retrieve store comment.
300
301 :return: Comment
302 """
303 raise NotImplementedError(self.get_comment)
304
305 def set_comment(self, comment):
306 """Set comment.
307
308 :param comment: New comment to set
309 """
310 raise NotImplementedError(self.set_comment)
311
312 def destroy(self):
313 """Destroy this store."""
314 raise NotImplementedError(self.destroy)
315
316 def subdirectories(self):
317 """Returns subdirectories to probe for other stores.
318
319 :return: List of names
320 """
321 raise NotImplementedError(self.subdirectories)
322
323
324 class GitStore(Store):
325 """A Store backed by a Git Repository.
326 """
327
328 def __init__(self, repo, ref=b'refs/heads/master',
329 check_for_duplicate_uids=True):
330 super(GitStore, self).__init__()
331 self.ref = ref
332 self.repo = repo
333 # Maps uids to (sha, fname)
334 self._uid_to_fname = {}
335 self._check_for_duplicate_uids = check_for_duplicate_uids
336 # Set of blob ids that have already been scanned
337 self._fname_to_uid = {}
338
339 def __repr__(self):
340 return "%s(%r, ref=%r)" % (type(self).__name__, self.repo, self.ref)
341
342 @property
343 def path(self):
344 return self.repo.path
345
346 def _check_duplicate(self, uid, name, replace_etag):
347 if uid is not None and self._check_for_duplicate_uids:
348 self._scan_uids()
349 try:
350 (existing_name, _) = self._uid_to_fname[uid]
351 except KeyError:
352 pass
353 else:
354 if existing_name != name:
355 raise DuplicateUidError(uid, existing_name, name)
356
357 try:
358 etag = self._get_etag(name)
359 except KeyError:
360 etag = None
361 if replace_etag is not None and etag != replace_etag:
362 raise InvalidETag(name, etag, replace_etag)
363 return etag
364
365 def import_one(self, name, content_type, data, message=None, author=None,
366 replace_etag=None):
367 """Import a single object.
368
369 :param name: name of the object
370 :param content_type: Content type
371 :param data: serialized object as list of bytes
372 :param message: Commit message
373 :param author: Optional author
374 :param replace_etag: optional etag of object to replace
375 :raise InvalidETag: when the name already exists but with different
376 etag
377 :raise DuplicateUidError: when the uid already exists
378 :return: etag
379 """
380 if content_type is None:
381 fi = open_by_extension(data, name, self.extra_file_handlers)
382 else:
383 fi = open_by_content_type(data, content_type, self.extra_file_handlers)
384 if name is None:
385 name = str(uuid.uuid4())
386 extension = MIMETYPES.guess_extension(content_type)
387 if extension is not None:
388 name += extension
389 fi.validate()
390 try:
391 uid = fi.get_uid()
392 except (KeyError, NotImplementedError):
393 uid = None
394 self._check_duplicate(uid, name, replace_etag)
395 if message is None:
396 try:
397 old_fi = self.get_file(name, content_type, replace_etag)
398 except KeyError:
399 old_fi = None
400 message = '\n'.join(fi.describe_delta(name, old_fi))
401 etag = self._import_one(name, data, message, author=author)
402 return (name, etag.decode('ascii'))
403
404 def _get_raw(self, name, etag=None):
405 """Get the raw contents of an object.
406
407 :param name: Name of the item
408 :param etag: Optional etag
409 :return: raw contents as chunks
410 """
411 if etag is None:
412 etag = self._get_etag(name)
413 blob = self.repo.object_store[etag.encode('ascii')]
414 return blob.chunked
415
416 def _scan_uids(self):
417 removed = set(self._fname_to_uid.keys())
418 for (name, mode, sha) in self._iterblobs():
419 etag = sha.decode('ascii')
420 if name in removed:
421 removed.remove(name)
422 if (name in self._fname_to_uid and
423 self._fname_to_uid[name][0] == etag):
424 continue
425 blob = self.repo.object_store[sha]
426 fi = open_by_extension(blob.chunked, name,
427 self.extra_file_handlers)
428 try:
429 uid = fi.get_uid()
430 except KeyError:
431 logger.warning('No UID found in file %s', name)
432 uid = None
433 except InvalidFileContents:
434 logging.warning('Unable to parse file %s', name)
435 uid = None
436 except NotImplementedError:
437 # This file type doesn't support UIDs
438 uid = None
439 self._fname_to_uid[name] = (etag, uid)
440 if uid is not None:
441 self._uid_to_fname[uid] = (name, etag)
442 for name in removed:
443 (unused_etag, uid) = self._fname_to_uid[name]
444 if uid is not None:
445 del self._uid_to_fname[uid]
446 del self._fname_to_uid[name]
447
448 def _iterblobs(self, ctag=None):
449 raise NotImplementedError(self._iterblobs)
450
451 def iter_with_etag(self, ctag=None):
452 """Iterate over all items in the store with etag.
453
454 :param ctag: Ctag to iterate for
455 :yield: (name, content_type, etag) tuples
456 """
457 for (name, mode, sha) in self._iterblobs(ctag):
458 (mime_type, _) = MIMETYPES.guess_type(name)
459 if mime_type is None:
460 mime_type = DEFAULT_MIME_TYPE
461 yield (name, mime_type, sha.decode('ascii'))
462
463 @classmethod
464 def create(cls, path):
465 """Create a new store backed by a Git repository on disk.
466
467 :return: A `GitStore`
468 """
469 raise NotImplementedError(cls.create)
470
471 @classmethod
472 def open_from_path(cls, path):
473 """Open a GitStore from a path.
474
475 :param path: Path
476 :return: A `GitStore`
477 """
478 try:
479 return cls.open(dulwich.repo.Repo(path))
480 except dulwich.repo.NotGitRepository:
481 raise NotStoreError(path)
482
483 @classmethod
484 def open(cls, repo):
485 """Open a GitStore given a Repo object.
486
487 :param repo: A Dulwich `Repo`
488 :return: A `GitStore`
489 """
490 if repo.has_index():
491 return TreeGitStore(repo)
492 else:
493 return BareGitStore(repo)
494
495 def get_description(self):
496 """Get extended description.
497
498 :return: repository description as string
499 """
500 desc = self.repo.get_description()
501 if desc is not None:
502 desc = desc.decode(DEFAULT_ENCODING)
503 return desc
504
505 def set_description(self, description):
506 """Set extended description.
507
508 :param description: repository description as string
509 """
510 return self.repo.set_description(description.encode(DEFAULT_ENCODING))
511
512 def set_comment(self, comment):
513 """Set comment.
514
515 :param comment: Comment
516 """
517 config = self.repo.get_config()
518 config.set(b'xandikos', b'comment', comment.encode(DEFAULT_ENCODING))
519 config.write_to_path()
520
521 def get_comment(self):
522 """Get comment.
523
524 :return: Comment
525 """
526 config = self.repo.get_config()
527 try:
528 comment = config.get(b'xandikos', b'comment')
529 except KeyError:
530 return None
531 else:
532 return comment.decode(DEFAULT_ENCODING)
533
534 def get_color(self):
535 """Get color.
536
537 :return: A Color code, or None
538 """
539 config = self.repo.get_config()
540 try:
541 color = config.get(b'xandikos', b'color')
542 except KeyError:
543 return None
544 else:
545 return color.decode(DEFAULT_ENCODING)
546
547 def set_color(self, color):
548 """Set the color code for this store."""
549 config = self.repo.get_config()
550 # Strip leading # to work around
551 # https://github.com/jelmer/dulwich/issues/511
552 # TODO(jelmer): Drop when that bug gets fixed.
553 config.set(
554 b'xandikos', b'color',
555 color.lstrip('#').encode(DEFAULT_ENCODING) if color else b'')
556 config.write_to_path()
557
558 def get_displayname(self):
559 """Get display name.
560
561 :return: The display name, or None if not set
562 """
563 config = self.repo.get_config()
564 try:
565 displayname = config.get(b'xandikos', b'displayname')
566 except KeyError:
567 return None
568 else:
569 return displayname.decode(DEFAULT_ENCODING)
570
571 def set_displayname(self, displayname):
572 """Set the display name.
573
574 :param displayname: New display name
575 """
576 config = self.repo.get_config()
577 config.set(b'xandikos', b'displayname',
578 displayname.encode(DEFAULT_ENCODING))
579 config.write_to_path()
580
581 def set_type(self, store_type):
582 """Set store type.
583
584 :param store_type: New store type (one of VALID_STORE_TYPES)
585 """
586 config = self.repo.get_config()
587 config.set(b'xandikos', b'type', store_type.encode(DEFAULT_ENCODING))
588 config.write_to_path()
589
590 def get_type(self):
591 """Get store type.
592
593 This looks in git config first, then falls back to guessing.
594 """
595 config = self.repo.get_config()
596 try:
597 store_type = config.get(b'xandikos', b'type')
598 except KeyError:
599 return super(GitStore, self).get_type()
600 else:
601 store_type = store_type.decode(DEFAULT_ENCODING)
602 if store_type not in VALID_STORE_TYPES:
603 logging.warning(
604 'Invalid store type %s set for %r.',
605 store_type, self.repo)
606 return store_type
607
608 def iter_changes(self, old_ctag, new_ctag):
609 """Get changes between two versions of this store.
610
611 :param old_ctag: Old ctag (None for empty Store)
612 :param new_ctag: New ctag
613 :return: Iterator over (name, content_type, old_etag, new_etag)
614 """
615 if old_ctag is None:
616 t = Tree()
617 self.repo.object_store.add_object(t)
618 old_ctag = t.id.decode('ascii')
619 previous = {
620 name: (content_type, etag)
621 for (name, content_type, etag) in self.iter_with_etag(old_ctag)
622 }
623 for (name, new_content_type, new_etag) in (
624 self.iter_with_etag(new_ctag)):
625 try:
626 (old_content_type, old_etag) = previous[name]
627 except KeyError:
628 old_etag = None
629 else:
630 assert old_content_type == new_content_type
631 if old_etag != new_etag:
632 yield (name, new_content_type, old_etag, new_etag)
633 if old_etag is not None:
634 del previous[name]
635 for (name, (old_content_type, old_etag)) in previous.items():
636 yield (name, old_content_type, old_etag, None)
637
638 def destroy(self):
639 """Destroy this store."""
640 shutil.rmtree(self.path)
641
642
643 class BareGitStore(GitStore):
644 """A Store backed by a bare git repository."""
645
646 def _get_current_tree(self):
647 try:
648 ref_object = self.repo[self.ref]
649 except KeyError:
650 return Tree()
651 if isinstance(ref_object, Tree):
652 return ref_object
653 else:
654 return self.repo.object_store[ref_object.tree]
655
656 def _get_etag(self, name):
657 tree = self._get_current_tree()
658 name = name.encode(DEFAULT_ENCODING)
659 return tree[name][1].decode('ascii')
660
661 def get_ctag(self):
662 """Return the ctag for this store."""
663 return self._get_current_tree().id.decode('ascii')
664
665 def _iterblobs(self, ctag=None):
666 if ctag is None:
667 tree = self._get_current_tree()
668 else:
669 tree = self.repo.object_store[ctag.encode('ascii')]
670 for (name, mode, sha) in tree.iteritems():
671 name = name.decode(DEFAULT_ENCODING)
672 yield (name, mode, sha)
673
674 @classmethod
675 def create_memory(cls):
676 """Create a new store backed by a memory repository.
677
678 :return: A `GitStore`
679 """
680 return cls(dulwich.repo.MemoryRepo())
681
682 def _commit_tree(self, tree_id, message, author=None):
683 try:
684 committer = self.repo._get_user_identity()
685 except KeyError:
686 committer = _DEFAULT_COMMITTER_IDENTITY
687 return self.repo.do_commit(message=message, tree=tree_id, ref=self.ref,
688 committer=committer, author=author)
689
690 def _import_one(self, name, data, message, author=None):
691 """Import a single object.
692
693 :param name: Optional name of the object
694 :param data: serialized object as bytes
695 :param message: optional commit message
696 :param author: optional author
697 :return: etag
698 """
699 b = Blob()
700 b.chunked = data
701 tree = self._get_current_tree()
702 name_enc = name.encode(DEFAULT_ENCODING)
703 tree[name_enc] = (0o644 | stat.S_IFREG, b.id)
704 self.repo.object_store.add_objects([(tree, ''), (b, name_enc)])
705 self._commit_tree(tree.id, message.encode(DEFAULT_ENCODING),
706 author=author)
707 return b.id
708
709 def delete_one(self, name, message=None, author=None, etag=None):
710 """Delete an item.
711
712 :param name: Filename to delete
713 :param message; Commit message
714 :param author: Optional author to store
715 :param etag: Optional mandatory etag of object to remove
716 :raise NoSuchItem: when the item doesn't exist
717 :raise InvalidETag: If the specified ETag doesn't match the curren
718 """
719 tree = self._get_current_tree()
720 name_enc = name.encode(DEFAULT_ENCODING)
721 try:
722 current_sha = tree[name_enc][1]
723 except KeyError:
724 raise NoSuchItem(name)
725 if etag is not None and current_sha != etag.encode('ascii'):
726 raise InvalidETag(name, etag, current_sha.decode('ascii'))
727 del tree[name_enc]
728 self.repo.object_store.add_objects([(tree, '')])
729 if message is None:
730 fi = open_by_extension(
731 self.repo.object_store[current_sha].chunked, name,
732 self.extra_file_handlers)
733 message = "Delete " + fi.describe(name)
734 self._commit_tree(tree.id, message.encode(DEFAULT_ENCODING),
735 author=author)
736
737 @classmethod
738 def create(cls, path):
739 """Create a new store backed by a Git repository on disk.
740
741 :return: A `GitStore`
742 """
743 os.mkdir(path)
744 return cls(dulwich.repo.Repo.init_bare(path))
745
746 def subdirectories(self):
747 """Returns subdirectories to probe for other stores.
748
749 :return: List of names
750 """
751 # Or perhaps just return all subdirectories but filter out
752 # Git-owned ones?
753 return []
754
755
756 class TreeGitStore(GitStore):
757 """A Store that backs onto a treefull Git repository."""
758
759 @classmethod
760 def create(cls, path, bare=True):
761 """Create a new store backed by a Git repository on disk.
762
763 :return: A `GitStore`
764 """
765 os.mkdir(path)
766 return cls(dulwich.repo.Repo.init(path))
767
768 def _get_etag(self, name):
769 index = self.repo.open_index()
770 name = name.encode(DEFAULT_ENCODING)
771 return index[name].sha.decode('ascii')
772
773 def _commit_tree(self, message, author=None):
774 try:
775 committer = self.repo._get_user_identity()
776 except KeyError:
777 committer = _DEFAULT_COMMITTER_IDENTITY
778 return self.repo.do_commit(message=message, committer=committer,
779 author=author)
780
781 def _import_one(self, name, data, message, author=None):
782 """Import a single object.
783
784 :param name: name of the object
785 :param data: serialized object as list of bytes
786 :param message: Commit message
787 :param author: Optional author
788 :return: etag
789 """
790 p = os.path.join(self.repo.path, name)
791 with open(p, 'wb') as f:
792 f.writelines(data)
793 self.repo.stage(name)
794 etag = self.repo.open_index()[name.encode(DEFAULT_ENCODING)].sha
795 self._commit_tree(message.encode(DEFAULT_ENCODING), author=author)
796 return etag
797
798 def delete_one(self, name, message=None, author=None, etag=None):
799 """Delete an item.
800
801 :param name: Filename to delete
802 :param message: Commit message
803 :param author: Optional author
804 :param etag: Optional mandatory etag of object to remove
805 :raise NoSuchItem: when the item doesn't exist
806 :raise InvalidETag: If the specified ETag doesn't match the curren
807 """
808 p = os.path.join(self.repo.path, name)
809 try:
810 with open(p, 'rb') as f:
811 current_blob = Blob.from_string(f.read())
812 except IOError:
813 raise NoSuchItem(name)
814 if message is None:
815 fi = open_by_extension(current_blob.chunked, name,
816 self.extra_file_handlers)
817 message = 'Delete ' + fi.describe(name)
818 if etag is not None:
819 with open(p, 'rb') as f:
820 current_etag = current_blob.id
821 if etag.encode('ascii') != current_etag:
822 raise InvalidETag(name, etag, current_etag.decode('ascii'))
823 os.unlink(p)
824 self.repo.stage(name)
825 self._commit_tree(message.encode(DEFAULT_ENCODING), author=author)
826
827 def get_ctag(self):
828 """Return the ctag for this store."""
829 index = self.repo.open_index()
830 return index.commit(self.repo.object_store).decode('ascii')
831
832 def _iterblobs(self, ctag=None):
833 """Iterate over all items in the store with etag.
834
835 :yield: (name, etag) tuples
836 """
837 if ctag is not None:
838 tree = self.repo.object_store[ctag.encode('ascii')]
839 for (name, mode, sha) in tree.iteritems():
840 name = name.decode(DEFAULT_ENCODING)
841 yield (name, mode, sha)
842 else:
843 index = self.repo.open_index()
844 for (name, sha, mode) in index.iterblobs():
845 name = name.decode(DEFAULT_ENCODING)
846 yield (name, mode, sha)
847
848 def subdirectories(self):
849 """Returns subdirectories to probe for other stores.
850
851 :return: List of names
852 """
853 ret = []
854 for name in os.listdir(self.path):
855 if name == dulwich.repo.CONTROLDIR:
856 continue
857 p = os.path.join(self.path, name)
858 if os.path.isdir(p):
859 ret.append(name)
860 return ret
861
862
863 def open_store(location):
864 """Open store from a location string.
865
866 :param location: Location string to open
867 :return: A `Store`
868 """
869 # For now, just support opening git stores
870 return GitStore.open_from_path(location)