1 # coding=utf-8
2 # (The line above is necessary so that I can use 世界 in the
3 # *comment* below without Python getting all bent out of shape.)
4 
5 # Copyright 2007-2009 Google Inc.
6 #
7 # Licensed under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
10 #
11 #	http://www.apache.org/licenses/LICENSE-2.0
12 #
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
18 
19 '''Mercurial interface to codereview.appspot.com.
20 
21 To configure, set the following options in
22 your repository's .hg/hgrc file.
23 
24 	[extensions]
25 	codereview = /path/to/codereview.py
26 
27 	[codereview]
28 	server = codereview.appspot.com
29 
30 The server should be running Rietveld; see http://code.google.com/p/rietveld/.
31 
32 In addition to the new commands, this extension introduces
33 the file pattern syntax @nnnnnn, where nnnnnn is a change list
34 number, to mean the files included in that change list, which
35 must be associated with the current client.
36 
37 For example, if change 123456 contains the files x.go and y.go,
38 "hg diff @123456" is equivalent to"hg diff x.go y.go".
39 '''
40 
41 import sys
42 
43 if __name__ == "__main__":
44 	print >>sys.stderr, "This is a Mercurial extension and should not be invoked directly."
45 	sys.exit(2)
46 
47 # We require Python 2.6 for the json package.
48 if sys.version < '2.6':
49 	print >>sys.stderr, "The codereview extension requires Python 2.6 or newer."
50 	print >>sys.stderr, "You are running Python " + sys.version
51 	sys.exit(2)
52 
53 import json
54 import os
55 import re
56 import stat
57 import subprocess
58 import threading
59 import time
60 
61 from mercurial import commands as hg_commands
62 from mercurial import util as hg_util
63 
64 defaultcc = None
65 codereview_disabled = None
66 real_rollback = None
67 releaseBranch = None
68 server = "codereview.appspot.com"
69 server_url_base = None
70 
71 #######################################################################
72 # Normally I would split this into multiple files, but it simplifies
73 # import path headaches to keep it all in one file.  Sorry.
74 # The different parts of the file are separated by banners like this one.
75 
76 #######################################################################
77 # Helpers
78 
79 def RelativePath(path, cwd):
80 	n = len(cwd)
81 	if path.startswith(cwd) and path[n] == '/':
82 		return path[n+1:]
83 	return path
84 
85 def Sub(l1, l2):
86 	return [l for l in l1 if l not in l2]
87 
88 def Add(l1, l2):
89 	l = l1 + Sub(l2, l1)
90 	l.sort()
91 	return l
92 
93 def Intersect(l1, l2):
94 	return [l for l in l1 if l in l2]
95 
96 #######################################################################
97 # RE: UNICODE STRING HANDLING
98 #
99 # Python distinguishes between the str (string of bytes)
100 # and unicode (string of code points) types.  Most operations
101 # work on either one just fine, but some (like regexp matching)
102 # require unicode, and others (like write) require str.
103 #
104 # As befits the language, Python hides the distinction between
105 # unicode and str by converting between them silently, but
106 # *only* if all the bytes/code points involved are 7-bit ASCII.
107 # This means that if you're not careful, your program works
108 # fine on "hello, world" and fails on "hello, 世界".  And of course,
109 # the obvious way to be careful - use static types - is unavailable.
110 # So the only way is trial and error to find where to put explicit
111 # conversions.
112 #
113 # Because more functions do implicit conversion to str (string of bytes)
114 # than do implicit conversion to unicode (string of code points),
115 # the convention in this module is to represent all text as str,
116 # converting to unicode only when calling a unicode-only function
117 # and then converting back to str as soon as possible.
118 
119 def typecheck(s, t):
120 	if type(s) != t:
121 		raise hg_util.Abort("type check failed: %s has type %s != %s" % (repr(s), type(s), t))
122 
123 # If we have to pass unicode instead of str, ustr does that conversion clearly.
124 def ustr(s):
125 	typecheck(s, str)
126 	return s.decode("utf-8")
127 
128 # Even with those, Mercurial still sometimes turns unicode into str
129 # and then tries to use it as ascii.  Change Mercurial's default.
130 def set_mercurial_encoding_to_utf8():
131 	from mercurial import encoding
132 	encoding.encoding = 'utf-8'
133 
134 set_mercurial_encoding_to_utf8()
135 
136 # Even with those we still run into problems.
137 # I tried to do things by the book but could not convince
138 # Mercurial to let me check in a change with UTF-8 in the
139 # CL description or author field, no matter how many conversions
140 # between str and unicode I inserted and despite changing the
141 # default encoding.  I'm tired of this game, so set the default
142 # encoding for all of Python to 'utf-8', not 'ascii'.
143 def default_to_utf8():
144 	import sys
145 	stdout, __stdout__ = sys.stdout, sys.__stdout__
146 	reload(sys)  # site.py deleted setdefaultencoding; get it back
147 	sys.stdout, sys.__stdout__ = stdout, __stdout__
148 	sys.setdefaultencoding('utf-8')
149 
150 default_to_utf8()
151 
152 #######################################################################
153 # Status printer for long-running commands
154 
155 global_status = None
156 
157 def set_status(s):
158 	# print >>sys.stderr, "\t", time.asctime(), s
159 	global global_status
160 	global_status = s
161 
162 class StatusThread(threading.Thread):
163 	def __init__(self):
164 		threading.Thread.__init__(self)
165 	def run(self):
166 		# pause a reasonable amount of time before
167 		# starting to display status messages, so that
168 		# most hg commands won't ever see them.
169 		time.sleep(30)
170 
171 		# now show status every 15 seconds
172 		while True:
173 			time.sleep(15 - time.time() % 15)
174 			s = global_status
175 			if s is None:
176 				continue
177 			if s == "":
178 				s = "(unknown status)"
179 			print >>sys.stderr, time.asctime(), s
180 
181 def start_status_thread():
182 	t = StatusThread()
183 	t.setDaemon(True)  # allowed to exit if t is still running
184 	t.start()
185 
186 #######################################################################
187 # Change list parsing.
188 #
189 # Change lists are stored in .hg/codereview/cl.nnnnnn
190 # where nnnnnn is the number assigned by the code review server.
191 # Most data about a change list is stored on the code review server
192 # too: the description, reviewer, and cc list are all stored there.
193 # The only thing in the cl.nnnnnn file is the list of relevant files.
194 # Also, the existence of the cl.nnnnnn file marks this repository
195 # as the one where the change list lives.
196 
197 emptydiff = """Index: ~rietveld~placeholder~
198 ===================================================================
199 diff --git a/~rietveld~placeholder~ b/~rietveld~placeholder~
200 new file mode 100644
201 """
202 
203 class CL(object):
204 	def __init__(self, name):
205 		typecheck(name, str)
206 		self.name = name
207 		self.desc = ''
208 		self.files = []
209 		self.reviewer = []
210 		self.cc = []
211 		self.url = ''
212 		self.local = False
213 		self.web = False
214 		self.copied_from = None	# None means current user
215 		self.mailed = False
216 		self.private = False
217 		self.lgtm = []
218 
219 	def DiskText(self):
220 		cl = self
221 		s = ""
222 		if cl.copied_from:
223 			s += "Author: " + cl.copied_from + "\n\n"
224 		if cl.private:
225 			s += "Private: " + str(self.private) + "\n"
226 		s += "Mailed: " + str(self.mailed) + "\n"
227 		s += "Description:\n"
228 		s += Indent(cl.desc, "\t")
229 		s += "Files:\n"
230 		for f in cl.files:
231 			s += "\t" + f + "\n"
232 		typecheck(s, str)
233 		return s
234 
235 	def EditorText(self):
236 		cl = self
237 		s = _change_prolog
238 		s += "\n"
239 		if cl.copied_from:
240 			s += "Author: " + cl.copied_from + "\n"
241 		if cl.url != '':
242 			s += 'URL: ' + cl.url + '	# cannot edit\n\n'
243 		if cl.private:
244 			s += "Private: True\n"
245 		s += "Reviewer: " + JoinComma(cl.reviewer) + "\n"
246 		s += "CC: " + JoinComma(cl.cc) + "\n"
247 		s += "\n"
248 		s += "Description:\n"
249 		if cl.desc == '':
250 			s += "\t<enter description here>\n"
251 		else:
252 			s += Indent(cl.desc, "\t")
253 		s += "\n"
254 		if cl.local or cl.name == "new":
255 			s += "Files:\n"
256 			for f in cl.files:
257 				s += "\t" + f + "\n"
258 			s += "\n"
259 		typecheck(s, str)
260 		return s
261 
262 	def PendingText(self, quick=False):
263 		cl = self
264 		s = cl.name + ":" + "\n"
265 		s += Indent(cl.desc, "\t")
266 		s += "\n"
267 		if cl.copied_from:
268 			s += "\tAuthor: " + cl.copied_from + "\n"
269 		if not quick:
270 			s += "\tReviewer: " + JoinComma(cl.reviewer) + "\n"
271 			for (who, line) in cl.lgtm:
272 				s += "\t\t" + who + ": " + line + "\n"
273 			s += "\tCC: " + JoinComma(cl.cc) + "\n"
274 		s += "\tFiles:\n"
275 		for f in cl.files:
276 			s += "\t\t" + f + "\n"
277 		typecheck(s, str)
278 		return s
279 
280 	def Flush(self, ui, repo):
281 		if self.name == "new":
282 			self.Upload(ui, repo, gofmt_just_warn=True, creating=True)
283 		dir = CodeReviewDir(ui, repo)
284 		path = dir + '/cl.' + self.name
285 		f = open(path+'!', "w")
286 		f.write(self.DiskText())
287 		f.close()
288 		if sys.platform == "win32" and os.path.isfile(path):
289 			os.remove(path)
290 		os.rename(path+'!', path)
291 		if self.web and not self.copied_from:
292 			EditDesc(self.name, desc=self.desc,
293 				reviewers=JoinComma(self.reviewer), cc=JoinComma(self.cc),
294 				private=self.private)
295 
296 	def Delete(self, ui, repo):
297 		dir = CodeReviewDir(ui, repo)
298 		os.unlink(dir + "/cl." + self.name)
299 
300 	def Subject(self):
301 		s = line1(self.desc)
302 		if len(s) > 60:
303 			s = s[0:55] + "..."
304 		if self.name != "new":
305 			s = "code review %s: %s" % (self.name, s)
306 		typecheck(s, str)
307 		return s
308 
309 	def Upload(self, ui, repo, send_mail=False, gofmt=True, gofmt_just_warn=False, creating=False, quiet=False):
310 		if not self.files and not creating:
311 			ui.warn("no files in change list\n")
312 		if ui.configbool("codereview", "force_gofmt", True) and gofmt:
313 			CheckFormat(ui, repo, self.files, just_warn=gofmt_just_warn)
314 		set_status("uploading CL metadata + diffs")
315 		os.chdir(repo.root)
316 		form_fields = [
317 			("content_upload", "1"),
318 			("reviewers", JoinComma(self.reviewer)),
319 			("cc", JoinComma(self.cc)),
320 			("description", self.desc),
321 			("base_hashes", ""),
322 		]
323 
324 		if self.name != "new":
325 			form_fields.append(("issue", self.name))
326 		vcs = None
327 		# We do not include files when creating the issue,
328 		# because we want the patch sets to record the repository
329 		# and base revision they are diffs against.  We use the patch
330 		# set message for that purpose, but there is no message with
331 		# the first patch set.  Instead the message gets used as the
332 		# new CL's overall subject.  So omit the diffs when creating
333 		# and then we'll run an immediate upload.
334 		# This has the effect that every CL begins with an empty "Patch set 1".
335 		if self.files and not creating:
336 			vcs = MercurialVCS(upload_options, ui, repo)
337 			data = vcs.GenerateDiff(self.files)
338 			files = vcs.GetBaseFiles(data)
339 			if len(data) > MAX_UPLOAD_SIZE:
340 				uploaded_diff_file = []
341 				form_fields.append(("separate_patches", "1"))
342 			else:
343 				uploaded_diff_file = [("data", "data.diff", data)]
344 		else:
345 			uploaded_diff_file = [("data", "data.diff", emptydiff)]
346 		
347 		if vcs and self.name != "new":
348 			form_fields.append(("subject", "diff -r " + vcs.base_rev + " " + ui.expandpath("default")))
349 		else:
350 			# First upload sets the subject for the CL itself.
351 			form_fields.append(("subject", self.Subject()))
352 		ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
353 		response_body = MySend("/upload", body, content_type=ctype)
354 		patchset = None
355 		msg = response_body
356 		lines = msg.splitlines()
357 		if len(lines) >= 2:
358 			msg = lines[0]
359 			patchset = lines[1].strip()
360 			patches = [x.split(" ", 1) for x in lines[2:]]
361 		if response_body.startswith("Issue updated.") and quiet:
362 			pass
363 		else:
364 			ui.status(msg + "\n")
365 		set_status("uploaded CL metadata + diffs")
366 		if not response_body.startswith("Issue created.") and not response_body.startswith("Issue updated."):
367 			raise hg_util.Abort("failed to update issue: " + response_body)
368 		issue = msg[msg.rfind("/")+1:]
369 		self.name = issue
370 		if not self.url:
371 			self.url = server_url_base + self.name
372 		if not uploaded_diff_file:
373 			set_status("uploading patches")
374 			patches = UploadSeparatePatches(issue, rpc, patchset, data, upload_options)
375 		if vcs:
376 			set_status("uploading base files")
377 			vcs.UploadBaseFiles(issue, rpc, patches, patchset, upload_options, files)
378 		if send_mail:
379 			set_status("sending mail")
380 			MySend("/" + issue + "/mail", payload="")
381 		self.web = True
382 		set_status("flushing changes to disk")
383 		self.Flush(ui, repo)
384 		return
385 
386 	def Mail(self, ui, repo):
387 		pmsg = "Hello " + JoinComma(self.reviewer)
388 		if self.cc:
389 			pmsg += " (cc: %s)" % (', '.join(self.cc),)
390 		pmsg += ",\n"
391 		pmsg += "\n"
392 		repourl = ui.expandpath("default")
393 		if not self.mailed:
394 			pmsg += "I'd like you to review this change to\n" + repourl + "\n"
395 		else:
396 			pmsg += "Please take another look.\n"
397 		typecheck(pmsg, str)
398 		PostMessage(ui, self.name, pmsg, subject=self.Subject())
399 		self.mailed = True
400 		self.Flush(ui, repo)
401 
402 def GoodCLName(name):
403 	typecheck(name, str)
404 	return re.match("^[0-9]+$", name)
405 
406 def ParseCL(text, name):
407 	typecheck(text, str)
408 	typecheck(name, str)
409 	sname = None
410 	lineno = 0
411 	sections = {
412 		'Author': '',
413 		'Description': '',
414 		'Files': '',
415 		'URL': '',
416 		'Reviewer': '',
417 		'CC': '',
418 		'Mailed': '',
419 		'Private': '',
420 	}
421 	for line in text.split('\n'):
422 		lineno += 1
423 		line = line.rstrip()
424 		if line != '' and line[0] == '#':
425 			continue
426 		if line == '' or line[0] == ' ' or line[0] == '\t':
427 			if sname == None and line != '':
428 				return None, lineno, 'text outside section'
429 			if sname != None:
430 				sections[sname] += line + '\n'
431 			continue
432 		p = line.find(':')
433 		if p >= 0:
434 			s, val = line[:p].strip(), line[p+1:].strip()
435 			if s in sections:
436 				sname = s
437 				if val != '':
438 					sections[sname] += val + '\n'
439 				continue
440 		return None, lineno, 'malformed section header'
441 
442 	for k in sections:
443 		sections[k] = StripCommon(sections[k]).rstrip()
444 
445 	cl = CL(name)
446 	if sections['Author']:
447 		cl.copied_from = sections['Author']
448 	cl.desc = sections['Description']
449 	for line in sections['Files'].split('\n'):
450 		i = line.find('#')
451 		if i >= 0:
452 			line = line[0:i].rstrip()
453 		line = line.strip()
454 		if line == '':
455 			continue
456 		cl.files.append(line)
457 	cl.reviewer = SplitCommaSpace(sections['Reviewer'])
458 	cl.cc = SplitCommaSpace(sections['CC'])
459 	cl.url = sections['URL']
460 	if sections['Mailed'] != 'False':
461 		# Odd default, but avoids spurious mailings when
462 		# reading old CLs that do not have a Mailed: line.
463 		# CLs created with this update will always have 
464 		# Mailed: False on disk.
465 		cl.mailed = True
466 	if sections['Private'] in ('True', 'true', 'Yes', 'yes'):
467 		cl.private = True
468 	if cl.desc == '<enter description here>':
469 		cl.desc = ''
470 	return cl, 0, ''
471 
472 def SplitCommaSpace(s):
473 	typecheck(s, str)
474 	s = s.strip()
475 	if s == "":
476 		return []
477 	return re.split(", *", s)
478 
479 def CutDomain(s):
480 	typecheck(s, str)
481 	i = s.find('@')
482 	if i >= 0:
483 		s = s[0:i]
484 	return s
485 
486 def JoinComma(l):
487 	for s in l:
488 		typecheck(s, str)
489 	return ", ".join(l)
490 
491 def ExceptionDetail():
492 	s = str(sys.exc_info()[0])
493 	if s.startswith("<type '") and s.endswith("'>"):
494 		s = s[7:-2]
495 	elif s.startswith("<class '") and s.endswith("'>"):
496 		s = s[8:-2]
497 	arg = str(sys.exc_info()[1])
498 	if len(arg) > 0:
499 		s += ": " + arg
500 	return s
501 
502 def IsLocalCL(ui, repo, name):
503 	return GoodCLName(name) and os.access(CodeReviewDir(ui, repo) + "/cl." + name, 0)
504 
505 # Load CL from disk and/or the web.
506 def LoadCL(ui, repo, name, web=True):
507 	typecheck(name, str)
508 	set_status("loading CL " + name)
509 	if not GoodCLName(name):
510 		return None, "invalid CL name"
511 	dir = CodeReviewDir(ui, repo)
512 	path = dir + "cl." + name
513 	if os.access(path, 0):
514 		ff = open(path)
515 		text = ff.read()
516 		ff.close()
517 		cl, lineno, err = ParseCL(text, name)
518 		if err != "":
519 			return None, "malformed CL data: "+err
520 		cl.local = True
521 	else:
522 		cl = CL(name)
523 	if web:
524 		set_status("getting issue metadata from web")
525 		d = JSONGet(ui, "/api/" + name + "?messages=true")
526 		set_status(None)
527 		if d is None:
528 			return None, "cannot load CL %s from server" % (name,)
529 		if 'owner_email' not in d or 'issue' not in d or str(d['issue']) != name:
530 			return None, "malformed response loading CL data from code review server"
531 		cl.dict = d
532 		cl.reviewer = d.get('reviewers', [])
533 		cl.cc = d.get('cc', [])
534 		if cl.local and cl.copied_from and cl.desc:
535 			# local copy of CL written by someone else
536 			# and we saved a description.  use that one,
537 			# so that committers can edit the description
538 			# before doing hg submit.
539 			pass
540 		else:
541 			cl.desc = d.get('description', "")
542 		cl.url = server_url_base + name
543 		cl.web = True
544 		cl.private = d.get('private', False) != False
545 		cl.lgtm = []
546 		for m in d.get('messages', []):
547 			if m.get('approval', False) == True:
548 				who = re.sub('@.*', '', m.get('sender', ''))
549 				text = re.sub("\n(.|\n)*", '', m.get('text', ''))
550 				cl.lgtm.append((who, text))
551 
552 	set_status("loaded CL " + name)
553 	return cl, ''
554 
555 class LoadCLThread(threading.Thread):
556 	def __init__(self, ui, repo, dir, f, web):
557 		threading.Thread.__init__(self)
558 		self.ui = ui
559 		self.repo = repo
560 		self.dir = dir
561 		self.f = f
562 		self.web = web
563 		self.cl = None
564 	def run(self):
565 		cl, err = LoadCL(self.ui, self.repo, self.f[3:], web=self.web)
566 		if err != '':
567 			self.ui.warn("loading "+self.dir+self.f+": " + err + "\n")
568 			return
569 		self.cl = cl
570 
571 # Load all the CLs from this repository.
572 def LoadAllCL(ui, repo, web=True):
573 	dir = CodeReviewDir(ui, repo)
574 	m = {}
575 	files = [f for f in os.listdir(dir) if f.startswith('cl.')]
576 	if not files:
577 		return m
578 	active = []
579 	first = True
580 	for f in files:
581 		t = LoadCLThread(ui, repo, dir, f, web)
582 		t.start()
583 		if web and first:
584 			# first request: wait in case it needs to authenticate
585 			# otherwise we get lots of user/password prompts
586 			# running in parallel.
587 			t.join()
588 			if t.cl:
589 				m[t.cl.name] = t.cl
590 			first = False
591 		else:
592 			active.append(t)
593 	for t in active:
594 		t.join()
595 		if t.cl:
596 			m[t.cl.name] = t.cl
597 	return m
598 
599 # Find repository root.  On error, ui.warn and return None
600 def RepoDir(ui, repo):
601 	url = repo.url();
602 	if not url.startswith('file:'):
603 		ui.warn("repository %s is not in local file system\n" % (url,))
604 		return None
605 	url = url[5:]
606 	if url.endswith('/'):
607 		url = url[:-1]
608 	typecheck(url, str)
609 	return url
610 
611 # Find (or make) code review directory.  On error, ui.warn and return None
612 def CodeReviewDir(ui, repo):
613 	dir = RepoDir(ui, repo)
614 	if dir == None:
615 		return None
616 	dir += '/.hg/codereview/'
617 	if not os.path.isdir(dir):
618 		try:
619 			os.mkdir(dir, 0700)
620 		except:
621 			ui.warn('cannot mkdir %s: %s\n' % (dir, ExceptionDetail()))
622 			return None
623 	typecheck(dir, str)
624 	return dir
625 
626 # Turn leading tabs into spaces, so that the common white space
627 # prefix doesn't get confused when people's editors write out 
628 # some lines with spaces, some with tabs.  Only a heuristic
629 # (some editors don't use 8 spaces either) but a useful one.
630 def TabsToSpaces(line):
631 	i = 0
632 	while i < len(line) and line[i] == '\t':
633 		i += 1
634 	return ' '*(8*i) + line[i:]
635 
636 # Strip maximal common leading white space prefix from text
637 def StripCommon(text):
638 	typecheck(text, str)
639 	ws = None
640 	for line in text.split('\n'):
641 		line = line.rstrip()
642 		if line == '':
643 			continue
644 		line = TabsToSpaces(line)
645 		white = line[:len(line)-len(line.lstrip())]
646 		if ws == None:
647 			ws = white
648 		else:
649 			common = ''
650 			for i in range(min(len(white), len(ws))+1):
651 				if white[0:i] == ws[0:i]:
652 					common = white[0:i]
653 			ws = common
654 		if ws == '':
655 			break
656 	if ws == None:
657 		return text
658 	t = ''
659 	for line in text.split('\n'):
660 		line = line.rstrip()
661 		line = TabsToSpaces(line)
662 		if line.startswith(ws):
663 			line = line[len(ws):]
664 		if line == '' and t == '':
665 			continue
666 		t += line + '\n'
667 	while len(t) >= 2 and t[-2:] == '\n\n':
668 		t = t[:-1]
669 	typecheck(t, str)
670 	return t
671 
672 # Indent text with indent.
673 def Indent(text, indent):
674 	typecheck(text, str)
675 	typecheck(indent, str)
676 	t = ''
677 	for line in text.split('\n'):
678 		t += indent + line + '\n'
679 	typecheck(t, str)
680 	return t
681 
682 # Return the first line of l
683 def line1(text):
684 	typecheck(text, str)
685 	return text.split('\n')[0]
686 
687 _change_prolog = """# Change list.
688 # Lines beginning with # are ignored.
689 # Multi-line values should be indented.
690 """
691 
692 desc_re = '^(.+: |(tag )?(release|weekly)\.|fix build|undo CL)'
693 
694 desc_msg = '''Your CL description appears not to use the standard form.
695 
696 The first line of your change description is conventionally a
697 one-line summary of the change, prefixed by the primary affected package,
698 and is used as the subject for code review mail; the rest of the description
699 elaborates.
700 
701 Examples:
702 
703 	encoding/rot13: new package
704 
705 	math: add IsInf, IsNaN
706 	
707 	net: fix cname in LookupHost
708 
709 	unicode: update to Unicode 5.0.2
710 
711 '''
712 
713 def promptyesno(ui, msg):
714 	return ui.promptchoice(msg, ["&yes", "&no"], 0) == 0
715 
716 def promptremove(ui, repo, f):
717 	if promptyesno(ui, "hg remove %s (y/n)?" % (f,)):
718 		if hg_commands.remove(ui, repo, 'path:'+f) != 0:
719 			ui.warn("error removing %s" % (f,))
720 
721 def promptadd(ui, repo, f):
722 	if promptyesno(ui, "hg add %s (y/n)?" % (f,)):
723 		if hg_commands.add(ui, repo, 'path:'+f) != 0:
724 			ui.warn("error adding %s" % (f,))
725 
726 def EditCL(ui, repo, cl):
727 	set_status(None)	# do not show status
728 	s = cl.EditorText()
729 	while True:
730 		s = ui.edit(s, ui.username())
731 		
732 		# We can't trust Mercurial + Python not to die before making the change,
733 		# so, by popular demand, just scribble the most recent CL edit into
734 		# $(hg root)/last-change so that if Mercurial does die, people
735 		# can look there for their work.
736 		try:
737 			f = open(repo.root+"/last-change", "w")
738 			f.write(s)
739 			f.close()
740 		except:
741 			pass
742 
743 		clx, line, err = ParseCL(s, cl.name)
744 		if err != '':
745 			if not promptyesno(ui, "error parsing change list: line %d: %s\nre-edit (y/n)?" % (line, err)):
746 				return "change list not modified"
747 			continue
748 		
749 		# Check description.
750 		if clx.desc == '':
751 			if promptyesno(ui, "change list should have a description\nre-edit (y/n)?"):
752 				continue
753 		elif re.search('<enter reason for undo>', clx.desc):
754 			if promptyesno(ui, "change list description omits reason for undo\nre-edit (y/n)?"):
755 				continue
756 		elif not re.match(desc_re, clx.desc.split('\n')[0]):
757 			if promptyesno(ui, desc_msg + "re-edit (y/n)?"):
758 				continue
759 
760 		# Check file list for files that need to be hg added or hg removed
761 		# or simply aren't understood.
762 		pats = ['path:'+f for f in clx.files]
763 		changed = hg_matchPattern(ui, repo, *pats, modified=True, added=True, removed=True)
764 		deleted = hg_matchPattern(ui, repo, *pats, deleted=True)
765 		unknown = hg_matchPattern(ui, repo, *pats, unknown=True)
766 		ignored = hg_matchPattern(ui, repo, *pats, ignored=True)
767 		clean = hg_matchPattern(ui, repo, *pats, clean=True)
768 		files = []
769 		for f in clx.files:
770 			if f in changed:
771 				files.append(f)
772 				continue
773 			if f in deleted:
774 				promptremove(ui, repo, f)
775 				files.append(f)
776 				continue
777 			if f in unknown:
778 				promptadd(ui, repo, f)
779 				files.append(f)
780 				continue
781 			if f in ignored:
782 				ui.warn("error: %s is excluded by .hgignore; omitting\n" % (f,))
783 				continue
784 			if f in clean:
785 				ui.warn("warning: %s is listed in the CL but unchanged\n" % (f,))
786 				files.append(f)
787 				continue
788 			p = repo.root + '/' + f
789 			if os.path.isfile(p):
790 				ui.warn("warning: %s is a file but not known to hg\n" % (f,))
791 				files.append(f)
792 				continue
793 			if os.path.isdir(p):
794 				ui.warn("error: %s is a directory, not a file; omitting\n" % (f,))
795 				continue
796 			ui.warn("error: %s does not exist; omitting\n" % (f,))
797 		clx.files = files
798 
799 		cl.desc = clx.desc
800 		cl.reviewer = clx.reviewer
801 		cl.cc = clx.cc
802 		cl.files = clx.files
803 		cl.private = clx.private
804 		break
805 	return ""
806 
807 # For use by submit, etc. (NOT by change)
808 # Get change list number or list of files from command line.
809 # If files are given, make a new change list.
810 def CommandLineCL(ui, repo, pats, opts, defaultcc=None):
811 	if len(pats) > 0 and GoodCLName(pats[0]):
812 		if len(pats) != 1:
813 			return None, "cannot specify change number and file names"
814 		if opts.get('message'):
815 			return None, "cannot use -m with existing CL"
816 		cl, err = LoadCL(ui, repo, pats[0], web=True)
817 		if err != "":
818 			return None, err
819 	else:
820 		cl = CL("new")
821 		cl.local = True
822 		cl.files = ChangedFiles(ui, repo, pats, taken=Taken(ui, repo))
823 		if not cl.files:
824 			return None, "no files changed"
825 	if opts.get('reviewer'):
826 		cl.reviewer = Add(cl.reviewer, SplitCommaSpace(opts.get('reviewer')))
827 	if opts.get('cc'):
828 		cl.cc = Add(cl.cc, SplitCommaSpace(opts.get('cc')))
829 	if defaultcc:
830 		cl.cc = Add(cl.cc, defaultcc)
831 	if cl.name == "new":
832 		if opts.get('message'):
833 			cl.desc = opts.get('message')
834 		else:
835 			err = EditCL(ui, repo, cl)
836 			if err != '':
837 				return None, err
838 	return cl, ""
839 
840 #######################################################################
841 # Change list file management
842 
843 # Return list of changed files in repository that match pats.
844 # The patterns came from the command line, so we warn
845 # if they have no effect or cannot be understood.
846 def ChangedFiles(ui, repo, pats, taken=None):
847 	taken = taken or {}
848 	# Run each pattern separately so that we can warn about
849 	# patterns that didn't do anything useful.
850 	for p in pats:
851 		for f in hg_matchPattern(ui, repo, p, unknown=True):
852 			promptadd(ui, repo, f)
853 		for f in hg_matchPattern(ui, repo, p, removed=True):
854 			promptremove(ui, repo, f)
855 		files = hg_matchPattern(ui, repo, p, modified=True, added=True, removed=True)
856 		for f in files:
857 			if f in taken:
858 				ui.warn("warning: %s already in CL %s\n" % (f, taken[f].name))
859 		if not files:
860 			ui.warn("warning: %s did not match any modified files\n" % (p,))
861 
862 	# Again, all at once (eliminates duplicates)
863 	l = hg_matchPattern(ui, repo, *pats, modified=True, added=True, removed=True)
864 	l.sort()
865 	if taken:
866 		l = Sub(l, taken.keys())
867 	return l
868 
869 # Return list of changed files in repository that match pats and still exist.
870 def ChangedExistingFiles(ui, repo, pats, opts):
871 	l = hg_matchPattern(ui, repo, *pats, modified=True, added=True)
872 	l.sort()
873 	return l
874 
875 # Return list of files claimed by existing CLs
876 def Taken(ui, repo):
877 	all = LoadAllCL(ui, repo, web=False)
878 	taken = {}
879 	for _, cl in all.items():
880 		for f in cl.files:
881 			taken[f] = cl
882 	return taken
883 
884 # Return list of changed files that are not claimed by other CLs
885 def DefaultFiles(ui, repo, pats):
886 	return ChangedFiles(ui, repo, pats, taken=Taken(ui, repo))
887 
888 #######################################################################
889 # File format checking.
890 
891 def CheckFormat(ui, repo, files, just_warn=False):
892 	set_status("running gofmt")
893 	CheckGofmt(ui, repo, files, just_warn)
894 	CheckTabfmt(ui, repo, files, just_warn)
895 
896 # Check that gofmt run on the list of files does not change them
897 def CheckGofmt(ui, repo, files, just_warn):
898 	files = gofmt_required(files)
899 	if not files:
900 		return
901 	cwd = os.getcwd()
902 	files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
903 	files = [f for f in files if os.access(f, 0)]
904 	if not files:
905 		return
906 	try:
907 		cmd = subprocess.Popen(["gofmt", "-l"] + files, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=sys.platform != "win32")
908 		cmd.stdin.close()
909 	except:
910 		raise hg_util.Abort("gofmt: " + ExceptionDetail())
911 	data = cmd.stdout.read()
912 	errors = cmd.stderr.read()
913 	cmd.wait()
914 	set_status("done with gofmt")
915 	if len(errors) > 0:
916 		ui.warn("gofmt errors:\n" + errors.rstrip() + "\n")
917 		return
918 	if len(data) > 0:
919 		msg = "gofmt needs to format these files (run hg gofmt):\n" + Indent(data, "\t").rstrip()
920 		if just_warn:
921 			ui.warn("warning: " + msg + "\n")
922 		else:
923 			raise hg_util.Abort(msg)
924 	return
925 
926 # Check that *.[chys] files indent using tabs.
927 def CheckTabfmt(ui, repo, files, just_warn):
928 	files = [f for f in files if f.startswith('src/') and re.search(r"\.[chys]$", f) and not re.search(r"\.tab\.[ch]$", f)]
929 	if not files:
930 		return
931 	cwd = os.getcwd()
932 	files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
933 	files = [f for f in files if os.access(f, 0)]
934 	badfiles = []
935 	for f in files:
936 		try:
937 			for line in open(f, 'r'):
938 				# Four leading spaces is enough to complain about,
939 				# except that some Plan 9 code uses four spaces as the label indent,
940 				# so allow that.
941 				if line.startswith('    ') and not re.match('    [A-Za-z0-9_]+:', line):
942 					badfiles.append(f)
943 					break
944 		except:
945 			# ignore cannot open file, etc.
946 			pass
947 	if len(badfiles) > 0:
948 		msg = "these files use spaces for indentation (use tabs instead):\n\t" + "\n\t".join(badfiles)
949 		if just_warn:
950 			ui.warn("warning: " + msg + "\n")
951 		else:
952 			raise hg_util.Abort(msg)
953 	return
954 
955 #######################################################################
956 # CONTRIBUTORS file parsing
957 
958 contributorsCache = None
959 contributorsURL = None
960 
961 def ReadContributors(ui, repo):
962 	global contributorsCache
963 	if contributorsCache is not None:
964 		return contributorsCache
965 
966 	try:
967 		if contributorsURL is not None:
968 			opening = contributorsURL
969 			f = urllib2.urlopen(contributorsURL)
970 		else:
971 			opening = repo.root + '/CONTRIBUTORS'
972 			f = open(repo.root + '/CONTRIBUTORS', 'r')
973 	except:
974 		ui.write("warning: cannot open %s: %s\n" % (opening, ExceptionDetail()))
975 		return
976 
977 	contributors = {}
978 	for line in f:
979 		# CONTRIBUTORS is a list of lines like:
980 		#	Person <email>
981 		#	Person <email> <alt-email>
982 		# The first email address is the one used in commit logs.
983 		if line.startswith('#'):
984 			continue
985 		m = re.match(r"([^<>]+\S)\s+(<[^<>\s]+>)((\s+<[^<>\s]+>)*)\s*$", line)
986 		if m:
987 			name = m.group(1)
988 			email = m.group(2)[1:-1]
989 			contributors[email.lower()] = (name, email)
990 			for extra in m.group(3).split():
991 				contributors[extra[1:-1].lower()] = (name, email)
992 
993 	contributorsCache = contributors
994 	return contributors
995 
996 def CheckContributor(ui, repo, user=None):
997 	set_status("checking CONTRIBUTORS file")
998 	user, userline = FindContributor(ui, repo, user, warn=False)
999 	if not userline:
1000 		raise hg_util.Abort("cannot find %s in CONTRIBUTORS" % (user,))
1001 	return userline
1002 
1003 def FindContributor(ui, repo, user=None, warn=True):
1004 	if not user:
1005 		user = ui.config("ui", "username")
1006 		if not user:
1007 			raise hg_util.Abort("[ui] username is not configured in .hgrc")
1008 	user = user.lower()
1009 	m = re.match(r".*<(.*)>", user)
1010 	if m:
1011 		user = m.group(1)
1012 
1013 	contributors = ReadContributors(ui, repo)
1014 	if user not in contributors:
1015 		if warn:
1016 			ui.warn("warning: cannot find %s in CONTRIBUTORS\n" % (user,))
1017 		return user, None
1018 	
1019 	user, email = contributors[user]
1020 	return email, "%s <%s>" % (user, email)
1021 
1022 #######################################################################
1023 # Mercurial helper functions.
1024 # Read http://mercurial.selenic.com/wiki/MercurialApi before writing any of these.
1025 # We use the ui.pushbuffer/ui.popbuffer + hg_commands.xxx tricks for all interaction
1026 # with Mercurial.  It has proved the most stable as they make changes.
1027 
1028 hgversion = hg_util.version()
1029 
1030 # We require Mercurial 1.9 and suggest Mercurial 2.0.
1031 # The details of the scmutil package changed then,
1032 # so allowing earlier versions would require extra band-aids below.
1033 # Ubuntu 11.10 ships with Mercurial 1.9.1 as the default version.
1034 hg_required = "1.9"
1035 hg_suggested = "2.0"
1036 
1037 old_message = """
1038 
1039 The code review extension requires Mercurial """+hg_required+""" or newer.
1040 You are using Mercurial """+hgversion+""".
1041 
1042 To install a new Mercurial, use
1043 
1044 	sudo easy_install mercurial=="""+hg_suggested+"""
1045 
1046 or visit http://mercurial.selenic.com/downloads/.
1047 """
1048 
1049 linux_message = """
1050 You may need to clear your current Mercurial installation by running:
1051 
1052 	sudo apt-get remove mercurial mercurial-common
1053 	sudo rm -rf /etc/mercurial
1054 """
1055 
1056 if hgversion < hg_required:
1057 	msg = old_message
1058 	if os.access("/etc/mercurial", 0):
1059 		msg += linux_message
1060 	raise hg_util.Abort(msg)
1061 
1062 from mercurial.hg import clean as hg_clean
1063 from mercurial import cmdutil as hg_cmdutil
1064 from mercurial import error as hg_error
1065 from mercurial import match as hg_match
1066 from mercurial import node as hg_node
1067 
1068 class uiwrap(object):
1069 	def __init__(self, ui):
1070 		self.ui = ui
1071 		ui.pushbuffer()
1072 		self.oldQuiet = ui.quiet
1073 		ui.quiet = True
1074 		self.oldVerbose = ui.verbose
1075 		ui.verbose = False
1076 	def output(self):
1077 		ui = self.ui
1078 		ui.quiet = self.oldQuiet
1079 		ui.verbose = self.oldVerbose
1080 		return ui.popbuffer()
1081 
1082 def to_slash(path):
1083 	if sys.platform == "win32":
1084 		return path.replace('\\', '/')
1085 	return path
1086 
1087 def hg_matchPattern(ui, repo, *pats, **opts):
1088 	w = uiwrap(ui)
1089 	hg_commands.status(ui, repo, *pats, **opts)
1090 	text = w.output()
1091 	ret = []
1092 	prefix = to_slash(os.path.realpath(repo.root))+'/'
1093 	for line in text.split('\n'):
1094 		f = line.split()
1095 		if len(f) > 1:
1096 			if len(pats) > 0:
1097 				# Given patterns, Mercurial shows relative to cwd
1098 				p = to_slash(os.path.realpath(f[1]))
1099 				if not p.startswith(prefix):
1100 					print >>sys.stderr, "File %s not in repo root %s.\n" % (p, prefix)
1101 				else:
1102 					ret.append(p[len(prefix):])
1103 			else:
1104 				# Without patterns, Mercurial shows relative to root (what we want)
1105 				ret.append(to_slash(f[1]))
1106 	return ret
1107 
1108 def hg_heads(ui, repo):
1109 	w = uiwrap(ui)
1110 	hg_commands.heads(ui, repo)
1111 	return w.output()
1112 
1113 noise = [
1114 	"",
1115 	"resolving manifests",
1116 	"searching for changes",
1117 	"couldn't find merge tool hgmerge",
1118 	"adding changesets",
1119 	"adding manifests",
1120 	"adding file changes",
1121 	"all local heads known remotely",
1122 ]
1123 
1124 def isNoise(line):
1125 	line = str(line)
1126 	for x in noise:
1127 		if line == x:
1128 			return True
1129 	return False
1130 
1131 def hg_incoming(ui, repo):
1132 	w = uiwrap(ui)
1133 	ret = hg_commands.incoming(ui, repo, force=False, bundle="")
1134 	if ret and ret != 1:
1135 		raise hg_util.Abort(ret)
1136 	return w.output()
1137 
1138 def hg_log(ui, repo, **opts):
1139 	for k in ['date', 'keyword', 'rev', 'user']:
1140 		if not opts.has_key(k):
1141 			opts[k] = ""
1142 	w = uiwrap(ui)
1143 	ret = hg_commands.log(ui, repo, **opts)
1144 	if ret:
1145 		raise hg_util.Abort(ret)
1146 	return w.output()
1147 
1148 def hg_outgoing(ui, repo, **opts):
1149 	w = uiwrap(ui)
1150 	ret = hg_commands.outgoing(ui, repo, **opts)
1151 	if ret and ret != 1:
1152 		raise hg_util.Abort(ret)
1153 	return w.output()
1154 
1155 def hg_pull(ui, repo, **opts):
1156 	w = uiwrap(ui)
1157 	ui.quiet = False
1158 	ui.verbose = True  # for file list
1159 	err = hg_commands.pull(ui, repo, **opts)
1160 	for line in w.output().split('\n'):
1161 		if isNoise(line):
1162 			continue
1163 		if line.startswith('moving '):
1164 			line = 'mv ' + line[len('moving '):]
1165 		if line.startswith('getting ') and line.find(' to ') >= 0:
1166 			line = 'mv ' + line[len('getting '):]
1167 		if line.startswith('getting '):
1168 			line = '+ ' + line[len('getting '):]
1169 		if line.startswith('removing '):
1170 			line = '- ' + line[len('removing '):]
1171 		ui.write(line + '\n')
1172 	return err
1173 
1174 def hg_push(ui, repo, **opts):
1175 	w = uiwrap(ui)
1176 	ui.quiet = False
1177 	ui.verbose = True
1178 	err = hg_commands.push(ui, repo, **opts)
1179 	for line in w.output().split('\n'):
1180 		if not isNoise(line):
1181 			ui.write(line + '\n')
1182 	return err
1183 
1184 def hg_commit(ui, repo, *pats, **opts):
1185 	return hg_commands.commit(ui, repo, *pats, **opts)
1186 
1187 #######################################################################
1188 # Mercurial precommit hook to disable commit except through this interface.
1189 
1190 commit_okay = False
1191 
1192 def precommithook(ui, repo, **opts):
1193 	if commit_okay:
1194 		return False  # False means okay.
1195 	ui.write("\ncodereview extension enabled; use mail, upload, or submit instead of commit\n\n")
1196 	return True
1197 
1198 #######################################################################
1199 # @clnumber file pattern support
1200 
1201 # We replace scmutil.match with the MatchAt wrapper to add the @clnumber pattern.
1202 
1203 match_repo = None
1204 match_ui = None
1205 match_orig = None
1206 
1207 def InstallMatch(ui, repo):
1208 	global match_repo
1209 	global match_ui
1210 	global match_orig
1211 
1212 	match_ui = ui
1213 	match_repo = repo
1214 
1215 	from mercurial import scmutil
1216 	match_orig = scmutil.match
1217 	scmutil.match = MatchAt
1218 
1219 def MatchAt(ctx, pats=None, opts=None, globbed=False, default='relpath'):
1220 	taken = []
1221 	files = []
1222 	pats = pats or []
1223 	opts = opts or {}
1224 	
1225 	for p in pats:
1226 		if p.startswith('@'):
1227 			taken.append(p)
1228 			clname = p[1:]
1229 			if clname == "default":
1230 				files = DefaultFiles(match_ui, match_repo, [])
1231 			else:
1232 				if not GoodCLName(clname):
1233 					raise hg_util.Abort("invalid CL name " + clname)
1234 				cl, err = LoadCL(match_repo.ui, match_repo, clname, web=False)
1235 				if err != '':
1236 					raise hg_util.Abort("loading CL " + clname + ": " + err)
1237 				if not cl.files:
1238 					raise hg_util.Abort("no files in CL " + clname)
1239 				files = Add(files, cl.files)
1240 	pats = Sub(pats, taken) + ['path:'+f for f in files]
1241 
1242 	# work-around for http://selenic.com/hg/rev/785bbc8634f8
1243 	if not hasattr(ctx, 'match'):
1244 		ctx = ctx[None]
1245 	return match_orig(ctx, pats=pats, opts=opts, globbed=globbed, default=default)
1246 
1247 #######################################################################
1248 # Commands added by code review extension.
1249 
1250 # As of Mercurial 2.1 the commands are all required to return integer
1251 # exit codes, whereas earlier versions allowed returning arbitrary strings
1252 # to be printed as errors.  We wrap the old functions to make sure we
1253 # always return integer exit codes now.  Otherwise Mercurial dies
1254 # with a TypeError traceback (unsupported operand type(s) for &: 'str' and 'int').
1255 # Introduce a Python decorator to convert old functions to the new
1256 # stricter convention.
1257 
1258 def hgcommand(f):
1259 	def wrapped(ui, repo, *pats, **opts):
1260 		err = f(ui, repo, *pats, **opts)
1261 		if type(err) is int:
1262 			return err
1263 		if not err:
1264 			return 0
1265 		raise hg_util.Abort(err)
1266 	wrapped.__doc__ = f.__doc__
1267 	return wrapped
1268 
1269 #######################################################################
1270 # hg change
1271 
1272 @hgcommand
1273 def change(ui, repo, *pats, **opts):
1274 	"""create, edit or delete a change list
1275 
1276 	Create, edit or delete a change list.
1277 	A change list is a group of files to be reviewed and submitted together,
1278 	plus a textual description of the change.
1279 	Change lists are referred to by simple alphanumeric names.
1280 
1281 	Changes must be reviewed before they can be submitted.
1282 
1283 	In the absence of options, the change command opens the
1284 	change list for editing in the default editor.
1285 
1286 	Deleting a change with the -d or -D flag does not affect
1287 	the contents of the files listed in that change.  To revert
1288 	the files listed in a change, use
1289 
1290 		hg revert @123456
1291 
1292 	before running hg change -d 123456.
1293 	"""
1294 
1295 	if codereview_disabled:
1296 		return codereview_disabled
1297 	
1298 	dirty = {}
1299 	if len(pats) > 0 and GoodCLName(pats[0]):
1300 		name = pats[0]
1301 		if len(pats) != 1:
1302 			return "cannot specify CL name and file patterns"
1303 		pats = pats[1:]
1304 		cl, err = LoadCL(ui, repo, name, web=True)
1305 		if err != '':
1306 			return err
1307 		if not cl.local and (opts["stdin"] or not opts["stdout"]):
1308 			return "cannot change non-local CL " + name
1309 	else:
1310 		name = "new"
1311 		cl = CL("new")
1312 		if repo[None].branch() != "default":
1313 			return "cannot create CL outside default branch; switch with 'hg update default'"
1314 		dirty[cl] = True
1315 		files = ChangedFiles(ui, repo, pats, taken=Taken(ui, repo))
1316 
1317 	if opts["delete"] or opts["deletelocal"]:
1318 		if opts["delete"] and opts["deletelocal"]:
1319 			return "cannot use -d and -D together"
1320 		flag = "-d"
1321 		if opts["deletelocal"]:
1322 			flag = "-D"
1323 		if name == "new":
1324 			return "cannot use "+flag+" with file patterns"
1325 		if opts["stdin"] or opts["stdout"]:
1326 			return "cannot use "+flag+" with -i or -o"
1327 		if not cl.local:
1328 			return "cannot change non-local CL " + name
1329 		if opts["delete"]:
1330 			if cl.copied_from:
1331 				return "original author must delete CL; hg change -D will remove locally"
1332 			PostMessage(ui, cl.name, "*** Abandoned ***", send_mail=cl.mailed)
1333 			EditDesc(cl.name, closed=True, private=cl.private)
1334 		cl.Delete(ui, repo)
1335 		return
1336 
1337 	if opts["stdin"]:
1338 		s = sys.stdin.read()
1339 		clx, line, err = ParseCL(s, name)
1340 		if err != '':
1341 			return "error parsing change list: line %d: %s" % (line, err)
1342 		if clx.desc is not None:
1343 			cl.desc = clx.desc;
1344 			dirty[cl] = True
1345 		if clx.reviewer is not None:
1346 			cl.reviewer = clx.reviewer
1347 			dirty[cl] = True
1348 		if clx.cc is not None:
1349 			cl.cc = clx.cc
1350 			dirty[cl] = True
1351 		if clx.files is not None:
1352 			cl.files = clx.files
1353 			dirty[cl] = True
1354 		if clx.private != cl.private:
1355 			cl.private = clx.private
1356 			dirty[cl] = True
1357 
1358 	if not opts["stdin"] and not opts["stdout"]:
1359 		if name == "new":
1360 			cl.files = files
1361 		err = EditCL(ui, repo, cl)
1362 		if err != "":
1363 			return err
1364 		dirty[cl] = True
1365 
1366 	for d, _ in dirty.items():
1367 		name = d.name
1368 		d.Flush(ui, repo)
1369 		if name == "new":
1370 			d.Upload(ui, repo, quiet=True)
1371 
1372 	if opts["stdout"]:
1373 		ui.write(cl.EditorText())
1374 	elif opts["pending"]:
1375 		ui.write(cl.PendingText())
1376 	elif name == "new":
1377 		if ui.quiet:
1378 			ui.write(cl.name)
1379 		else:
1380 			ui.write("CL created: " + cl.url + "\n")
1381 	return
1382 
1383 #######################################################################
1384 # hg code-login (broken?)
1385 
1386 @hgcommand
1387 def code_login(ui, repo, **opts):
1388 	"""log in to code review server
1389 
1390 	Logs in to the code review server, saving a cookie in
1391 	a file in your home directory.
1392 	"""
1393 	if codereview_disabled:
1394 		return codereview_disabled
1395 
1396 	MySend(None)
1397 
1398 #######################################################################
1399 # hg clpatch / undo / release-apply / download
1400 # All concerned with applying or unapplying patches to the repository.
1401 
1402 @hgcommand
1403 def clpatch(ui, repo, clname, **opts):
1404 	"""import a patch from the code review server
1405 
1406 	Imports a patch from the code review server into the local client.
1407 	If the local client has already modified any of the files that the
1408 	patch modifies, this command will refuse to apply the patch.
1409 
1410 	Submitting an imported patch will keep the original author's
1411 	name as the Author: line but add your own name to a Committer: line.
1412 	"""
1413 	if repo[None].branch() != "default":
1414 		return "cannot run hg clpatch outside default branch"
1415 	return clpatch_or_undo(ui, repo, clname, opts, mode="clpatch")
1416 
1417 @hgcommand
1418 def undo(ui, repo, clname, **opts):
1419 	"""undo the effect of a CL
1420 	
1421 	Creates a new CL that undoes an earlier CL.
1422 	After creating the CL, opens the CL text for editing so that
1423 	you can add the reason for the undo to the description.
1424 	"""
1425 	if repo[None].branch() != "default":
1426 		return "cannot run hg undo outside default branch"
1427 	return clpatch_or_undo(ui, repo, clname, opts, mode="undo")
1428 
1429 @hgcommand
1430 def release_apply(ui, repo, clname, **opts):
1431 	"""apply a CL to the release branch
1432 
1433 	Creates a new CL copying a previously committed change
1434 	from the main branch to the release branch.
1435 	The current client must either be clean or already be in
1436 	the release branch.
1437 	
1438 	The release branch must be created by starting with a
1439 	clean client, disabling the code review plugin, and running:
1440 	
1441 		hg update weekly.YYYY-MM-DD
1442 		hg branch release-branch.rNN
1443 		hg commit -m 'create release-branch.rNN'
1444 		hg push --new-branch
1445 	
1446 	Then re-enable the code review plugin.
1447 	
1448 	People can test the release branch by running
1449 	
1450 		hg update release-branch.rNN
1451 	
1452 	in a clean client.  To return to the normal tree,
1453 	
1454 		hg update default
1455 	
1456 	Move changes since the weekly into the release branch 
1457 	using hg release-apply followed by the usual code review
1458 	process and hg submit.
1459 
1460 	When it comes time to tag the release, record the
1461 	final long-form tag of the release-branch.rNN
1462 	in the *default* branch's .hgtags file.  That is, run
1463 	
1464 		hg update default
1465 	
1466 	and then edit .hgtags as you would for a weekly.
1467 		
1468 	"""
1469 	c = repo[None]
1470 	if not releaseBranch:
1471 		return "no active release branches"
1472 	if c.branch() != releaseBranch:
1473 		if c.modified() or c.added() or c.removed():
1474 			raise hg_util.Abort("uncommitted local changes - cannot switch branches")
1475 		err = hg_clean(repo, releaseBranch)
1476 		if err:
1477 			return err
1478 	try:
1479 		err = clpatch_or_undo(ui, repo, clname, opts, mode="backport")
1480 		if err:
1481 			raise hg_util.Abort(err)
1482 	except Exception, e:
1483 		hg_clean(repo, "default")
1484 		raise e
1485 	return None
1486 
1487 def rev2clname(rev):
1488 	# Extract CL name from revision description.
1489 	# The last line in the description that is a codereview URL is the real one.
1490 	# Earlier lines might be part of the user-written description.
1491 	all = re.findall('(?m)^http://codereview.appspot.com/([0-9]+)$', rev.description())
1492 	if len(all) > 0:
1493 		return all[-1]
1494 	return ""
1495 
1496 undoHeader = """undo CL %s / %s
1497 
1498 <enter reason for undo>
1499 
1500 ««« original CL description
1501 """
1502 
1503 undoFooter = """
1504 »»»
1505 """
1506 
1507 backportHeader = """[%s] %s
1508 
1509 ««« CL %s / %s
1510 """
1511 
1512 backportFooter = """
1513 »»»
1514 """
1515 
1516 # Implementation of clpatch/undo.
1517 def clpatch_or_undo(ui, repo, clname, opts, mode):
1518 	if codereview_disabled:
1519 		return codereview_disabled
1520 
1521 	if mode == "undo" or mode == "backport":
1522 		# Find revision in Mercurial repository.
1523 		# Assume CL number is 7+ decimal digits.
1524 		# Otherwise is either change log sequence number (fewer decimal digits),
1525 		# hexadecimal hash, or tag name.
1526 		# Mercurial will fall over long before the change log
1527 		# sequence numbers get to be 7 digits long.
1528 		if re.match('^[0-9]{7,}$', clname):
1529 			found = False
1530 			for r in hg_log(ui, repo, keyword="codereview.appspot.com/"+clname, limit=100, template="{node}\n").split():
1531 				rev = repo[r]
1532 				# Last line with a code review URL is the actual review URL.
1533 				# Earlier ones might be part of the CL description.
1534 				n = rev2clname(rev)
1535 				if n == clname:
1536 					found = True
1537 					break
1538 			if not found:
1539 				return "cannot find CL %s in local repository" % clname
1540 		else:
1541 			rev = repo[clname]
1542 			if not rev:
1543 				return "unknown revision %s" % clname
1544 			clname = rev2clname(rev)
1545 			if clname == "":
1546 				return "cannot find CL name in revision description"
1547 		
1548 		# Create fresh CL and start with patch that would reverse the change.
1549 		vers = hg_node.short(rev.node())
1550 		cl = CL("new")
1551 		desc = str(rev.description())
1552 		if mode == "undo":
1553 			cl.desc = (undoHeader % (clname, vers)) + desc + undoFooter
1554 		else:
1555 			cl.desc = (backportHeader % (releaseBranch, line1(desc), clname, vers)) + desc + undoFooter
1556 		v1 = vers
1557 		v0 = hg_node.short(rev.parents()[0].node())
1558 		if mode == "undo":
1559 			arg = v1 + ":" + v0
1560 		else:
1561 			vers = v0
1562 			arg = v0 + ":" + v1
1563 		patch = RunShell(["hg", "diff", "--git", "-r", arg])
1564 
1565 	else:  # clpatch
1566 		cl, vers, patch, err = DownloadCL(ui, repo, clname)
1567 		if err != "":
1568 			return err
1569 		if patch == emptydiff:
1570 			return "codereview issue %s has no diff" % clname
1571 
1572 	# find current hg version (hg identify)
1573 	ctx = repo[None]
1574 	parents = ctx.parents()
1575 	id = '+'.join([hg_node.short(p.node()) for p in parents])
1576 
1577 	# if version does not match the patch version,
1578 	# try to update the patch line numbers.
1579 	if vers != "" and id != vers:
1580 		# "vers in repo" gives the wrong answer
1581 		# on some versions of Mercurial.  Instead, do the actual
1582 		# lookup and catch the exception.
1583 		try:
1584 			repo[vers].description()
1585 		except:
1586 			return "local repository is out of date; sync to get %s" % (vers)
1587 		patch1, err = portPatch(repo, patch, vers, id)
1588 		if err != "":
1589 			if not opts["ignore_hgpatch_failure"]:
1590 				return "codereview issue %s is out of date: %s (%s->%s)" % (clname, err, vers, id)
1591 		else:
1592 			patch = patch1
1593 	argv = ["hgpatch"]
1594 	if opts["no_incoming"] or mode == "backport":
1595 		argv += ["--checksync=false"]
1596 	try:
1597 		cmd = subprocess.Popen(argv, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, close_fds=sys.platform != "win32")
1598 	except:
1599 		return "hgpatch: " + ExceptionDetail() + "\nInstall hgpatch with:\n$ go get code.google.com/p/go.codereview/cmd/hgpatch\n"
1600 
1601 	out, err = cmd.communicate(patch)
1602 	if cmd.returncode != 0 and not opts["ignore_hgpatch_failure"]:
1603 		return "hgpatch failed"
1604 	cl.local = True
1605 	cl.files = out.strip().split()
1606 	if not cl.files and not opts["ignore_hgpatch_failure"]:
1607 		return "codereview issue %s has no changed files" % clname
1608 	files = ChangedFiles(ui, repo, [])
1609 	extra = Sub(cl.files, files)
1610 	if extra:
1611 		ui.warn("warning: these files were listed in the patch but not changed:\n\t" + "\n\t".join(extra) + "\n")
1612 	cl.Flush(ui, repo)
1613 	if mode == "undo":
1614 		err = EditCL(ui, repo, cl)
1615 		if err != "":
1616 			return "CL created, but error editing: " + err
1617 		cl.Flush(ui, repo)
1618 	else:
1619 		ui.write(cl.PendingText() + "\n")
1620 
1621 # portPatch rewrites patch from being a patch against
1622 # oldver to being a patch against newver.
1623 def portPatch(repo, patch, oldver, newver):
1624 	lines = patch.splitlines(True) # True = keep \n
1625 	delta = None
1626 	for i in range(len(lines)):
1627 		line = lines[i]
1628 		if line.startswith('--- a/'):
1629 			file = line[6:-1]
1630 			delta = fileDeltas(repo, file, oldver, newver)
1631 		if not delta or not line.startswith('@@ '):
1632 			continue
1633 		# @@ -x,y +z,w @@ means the patch chunk replaces
1634 		# the original file's line numbers x up to x+y with the
1635 		# line numbers z up to z+w in the new file.
1636 		# Find the delta from x in the original to the same
1637 		# line in the current version and add that delta to both
1638 		# x and z.
1639 		m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line)
1640 		if not m:
1641 			return None, "error parsing patch line numbers"
1642 		n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))
1643 		d, err = lineDelta(delta, n1, len1)
1644 		if err != "":
1645 			return "", err
1646 		n1 += d
1647 		n2 += d
1648 		lines[i] = "@@ -%d,%d +%d,%d @@\n" % (n1, len1, n2, len2)
1649 		
1650 	newpatch = ''.join(lines)
1651 	return newpatch, ""
1652 
1653 # fileDelta returns the line number deltas for the given file's
1654 # changes from oldver to newver.
1655 # The deltas are a list of (n, len, newdelta) triples that say
1656 # lines [n, n+len) were modified, and after that range the
1657 # line numbers are +newdelta from what they were before.
1658 def fileDeltas(repo, file, oldver, newver):
1659 	cmd = ["hg", "diff", "--git", "-r", oldver + ":" + newver, "path:" + file]
1660 	data = RunShell(cmd, silent_ok=True)
1661 	deltas = []
1662 	for line in data.splitlines():
1663 		m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line)
1664 		if not m:
1665 			continue
1666 		n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))
1667 		deltas.append((n1, len1, n2+len2-(n1+len1)))
1668 	return deltas
1669 
1670 # lineDelta finds the appropriate line number delta to apply to the lines [n, n+len).
1671 # It returns an error if those lines were rewritten by the patch.
1672 def lineDelta(deltas, n, len):
1673 	d = 0
1674 	for (old, oldlen, newdelta) in deltas:
1675 		if old >= n+len:
1676 			break
1677 		if old+len > n:
1678 			return 0, "patch and recent changes conflict"
1679 		d = newdelta
1680 	return d, ""
1681 
1682 @hgcommand
1683 def download(ui, repo, clname, **opts):
1684 	"""download a change from the code review server
1685 
1686 	Download prints a description of the given change list
1687 	followed by its diff, downloaded from the code review server.
1688 	"""
1689 	if codereview_disabled:
1690 		return codereview_disabled
1691 
1692 	cl, vers, patch, err = DownloadCL(ui, repo, clname)
1693 	if err != "":
1694 		return err
1695 	ui.write(cl.EditorText() + "\n")
1696 	ui.write(patch + "\n")
1697 	return
1698 
1699 #######################################################################
1700 # hg file
1701 
1702 @hgcommand
1703 def file(ui, repo, clname, pat, *pats, **opts):
1704 	"""assign files to or remove files from a change list
1705 
1706 	Assign files to or (with -d) remove files from a change list.
1707 
1708 	The -d option only removes files from the change list.
1709 	It does not edit them or remove them from the repository.
1710 	"""
1711 	if codereview_disabled:
1712 		return codereview_disabled
1713 
1714 	pats = tuple([pat] + list(pats))
1715 	if not GoodCLName(clname):
1716 		return "invalid CL name " + clname
1717 
1718 	dirty = {}
1719 	cl, err = LoadCL(ui, repo, clname, web=False)
1720 	if err != '':
1721 		return err
1722 	if not cl.local:
1723 		return "cannot change non-local CL " + clname
1724 
1725 	files = ChangedFiles(ui, repo, pats)
1726 
1727 	if opts["delete"]:
1728 		oldfiles = Intersect(files, cl.files)
1729 		if oldfiles:
1730 			if not ui.quiet:
1731 				ui.status("# Removing files from CL.  To undo:\n")
1732 				ui.status("#	cd %s\n" % (repo.root))
1733 				for f in oldfiles:
1734 					ui.status("#	hg file %s %s\n" % (cl.name, f))
1735 			cl.files = Sub(cl.files, oldfiles)
1736 			cl.Flush(ui, repo)
1737 		else:
1738 			ui.status("no such files in CL")
1739 		return
1740 
1741 	if not files:
1742 		return "no such modified files"
1743 
1744 	files = Sub(files, cl.files)
1745 	taken = Taken(ui, repo)
1746 	warned = False
1747 	for f in files:
1748 		if f in taken:
1749 			if not warned and not ui.quiet:
1750 				ui.status("# Taking files from other CLs.  To undo:\n")
1751 				ui.status("#	cd %s\n" % (repo.root))
1752 				warned = True
1753 			ocl = taken[f]
1754 			if not ui.quiet:
1755 				ui.status("#	hg file %s %s\n" % (ocl.name, f))
1756 			if ocl not in dirty:
1757 				ocl.files = Sub(ocl.files, files)
1758 				dirty[ocl] = True
1759 	cl.files = Add(cl.files, files)
1760 	dirty[cl] = True
1761 	for d, _ in dirty.items():
1762 		d.Flush(ui, repo)
1763 	return
1764 
1765 #######################################################################
1766 # hg gofmt
1767 
1768 @hgcommand
1769 def gofmt(ui, repo, *pats, **opts):
1770 	"""apply gofmt to modified files
1771 
1772 	Applies gofmt to the modified files in the repository that match
1773 	the given patterns.
1774 	"""
1775 	if codereview_disabled:
1776 		return codereview_disabled
1777 
1778 	files = ChangedExistingFiles(ui, repo, pats, opts)
1779 	files = gofmt_required(files)
1780 	if not files:
1781 		return "no modified go files"
1782 	cwd = os.getcwd()
1783 	files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
1784 	try:
1785 		cmd = ["gofmt", "-l"]
1786 		if not opts["list"]:
1787 			cmd += ["-w"]
1788 		if os.spawnvp(os.P_WAIT, "gofmt", cmd + files) != 0:
1789 			raise hg_util.Abort("gofmt did not exit cleanly")
1790 	except hg_error.Abort, e:
1791 		raise
1792 	except:
1793 		raise hg_util.Abort("gofmt: " + ExceptionDetail())
1794 	return
1795 
1796 def gofmt_required(files):
1797 	return [f for f in files if (not f.startswith('test/') or f.startswith('test/bench/')) and f.endswith('.go')]
1798 
1799 #######################################################################
1800 # hg mail
1801 
1802 @hgcommand
1803 def mail(ui, repo, *pats, **opts):
1804 	"""mail a change for review
1805 
1806 	Uploads a patch to the code review server and then sends mail
1807 	to the reviewer and CC list asking for a review.
1808 	"""
1809 	if codereview_disabled:
1810 		return codereview_disabled
1811 
1812 	cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
1813 	if err != "":
1814 		return err
1815 	cl.Upload(ui, repo, gofmt_just_warn=True)
1816 	if not cl.reviewer:
1817 		# If no reviewer is listed, assign the review to defaultcc.
1818 		# This makes sure that it appears in the 
1819 		# codereview.appspot.com/user/defaultcc
1820 		# page, so that it doesn't get dropped on the floor.
1821 		if not defaultcc:
1822 			return "no reviewers listed in CL"
1823 		cl.cc = Sub(cl.cc, defaultcc)
1824 		cl.reviewer = defaultcc
1825 		cl.Flush(ui, repo)
1826 
1827 	if cl.files == []:
1828 		return "no changed files, not sending mail"
1829 
1830 	cl.Mail(ui, repo)		
1831 
1832 #######################################################################
1833 # hg p / hg pq / hg ps / hg pending
1834 
1835 @hgcommand
1836 def ps(ui, repo, *pats, **opts):
1837 	"""alias for hg p --short
1838 	"""
1839 	opts['short'] = True
1840 	return pending(ui, repo, *pats, **opts)
1841 
1842 @hgcommand
1843 def pq(ui, repo, *pats, **opts):
1844 	"""alias for hg p --quick
1845 	"""
1846 	opts['quick'] = True
1847 	return pending(ui, repo, *pats, **opts)
1848 
1849 @hgcommand
1850 def pending(ui, repo, *pats, **opts):
1851 	"""show pending changes
1852 
1853 	Lists pending changes followed by a list of unassigned but modified files.
1854 	"""
1855 	if codereview_disabled:
1856 		return codereview_disabled
1857 
1858 	quick = opts.get('quick', False)
1859 	short = opts.get('short', False)
1860 	m = LoadAllCL(ui, repo, web=not quick and not short)
1861 	names = m.keys()
1862 	names.sort()
1863 	for name in names:
1864 		cl = m[name]
1865 		if short:
1866 			ui.write(name + "\t" + line1(cl.desc) + "\n")
1867 		else:
1868 			ui.write(cl.PendingText(quick=quick) + "\n")
1869 
1870 	if short:
1871 		return
1872 	files = DefaultFiles(ui, repo, [])
1873 	if len(files) > 0:
1874 		s = "Changed files not in any CL:\n"
1875 		for f in files:
1876 			s += "\t" + f + "\n"
1877 		ui.write(s)
1878 
1879 #######################################################################
1880 # hg submit
1881 
1882 def need_sync():
1883 	raise hg_util.Abort("local repository out of date; must sync before submit")
1884 
1885 @hgcommand
1886 def submit(ui, repo, *pats, **opts):
1887 	"""submit change to remote repository
1888 
1889 	Submits change to remote repository.
1890 	Bails out if the local repository is not in sync with the remote one.
1891 	"""
1892 	if codereview_disabled:
1893 		return codereview_disabled
1894 
1895 	# We already called this on startup but sometimes Mercurial forgets.
1896 	set_mercurial_encoding_to_utf8()
1897 
1898 	if not opts["no_incoming"] and hg_incoming(ui, repo):
1899 		need_sync()
1900 
1901 	cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
1902 	if err != "":
1903 		return err
1904 
1905 	user = None
1906 	if cl.copied_from:
1907 		user = cl.copied_from
1908 	userline = CheckContributor(ui, repo, user)
1909 	typecheck(userline, str)
1910 
1911 	about = ""
1912 	if cl.reviewer:
1913 		about += "R=" + JoinComma([CutDomain(s) for s in cl.reviewer]) + "\n"
1914 	if opts.get('tbr'):
1915 		tbr = SplitCommaSpace(opts.get('tbr'))
1916 		cl.reviewer = Add(cl.reviewer, tbr)
1917 		about += "TBR=" + JoinComma([CutDomain(s) for s in tbr]) + "\n"
1918 	if cl.cc:
1919 		about += "CC=" + JoinComma([CutDomain(s) for s in cl.cc]) + "\n"
1920 
1921 	if not cl.reviewer:
1922 		return "no reviewers listed in CL"
1923 
1924 	if not cl.local:
1925 		return "cannot submit non-local CL"
1926 
1927 	# upload, to sync current patch and also get change number if CL is new.
1928 	if not cl.copied_from:
1929 		cl.Upload(ui, repo, gofmt_just_warn=True)
1930 
1931 	# check gofmt for real; allowed upload to warn in order to save CL.
1932 	cl.Flush(ui, repo)
1933 	CheckFormat(ui, repo, cl.files)
1934 
1935 	about += "%s%s\n" % (server_url_base, cl.name)
1936 
1937 	if cl.copied_from:
1938 		about += "\nCommitter: " + CheckContributor(ui, repo, None) + "\n"
1939 	typecheck(about, str)
1940 
1941 	if not cl.mailed and not cl.copied_from:		# in case this is TBR
1942 		cl.Mail(ui, repo)
1943 
1944 	# submit changes locally
1945 	message = cl.desc.rstrip() + "\n\n" + about
1946 	typecheck(message, str)
1947 
1948 	set_status("pushing " + cl.name + " to remote server")
1949 
1950 	if hg_outgoing(ui, repo):
1951 		raise hg_util.Abort("local repository corrupt or out-of-phase with remote: found outgoing changes")
1952 	
1953 	old_heads = len(hg_heads(ui, repo).split())
1954 
1955 	global commit_okay
1956 	commit_okay = True
1957 	ret = hg_commit(ui, repo, *['path:'+f for f in cl.files], message=message, user=userline)
1958 	commit_okay = False
1959 	if ret:
1960 		return "nothing changed"
1961 	node = repo["-1"].node()
1962 	# push to remote; if it fails for any reason, roll back
1963 	try:
1964 		new_heads = len(hg_heads(ui, repo).split())
1965 		if old_heads != new_heads and not (old_heads == 0 and new_heads == 1):
1966 			# Created new head, so we weren't up to date.
1967 			need_sync()
1968 
1969 		# Push changes to remote.  If it works, we're committed.  If not, roll back.
1970 		try:
1971 			hg_push(ui, repo)
1972 		except hg_error.Abort, e:
1973 			if e.message.find("push creates new heads") >= 0:
1974 				# Remote repository had changes we missed.
1975 				need_sync()
1976 			raise
1977 	except:
1978 		real_rollback()
1979 		raise
1980 
1981 	# We're committed. Upload final patch, close review, add commit message.
1982 	changeURL = hg_node.short(node)
1983 	url = ui.expandpath("default")
1984 	m = re.match("(^https?://([^@/]+@)?([^.]+)\.googlecode\.com/hg/?)" + "|" +
1985 		"(^https?://([^@/]+@)?code\.google\.com/p/([^/.]+)(\.[^./]+)?/?)", url)
1986 	if m:
1987 		if m.group(1): # prj.googlecode.com/hg/ case
1988 			changeURL = "http://code.google.com/p/%s/source/detail?r=%s" % (m.group(3), changeURL)
1989 		elif m.group(4) and m.group(7): # code.google.com/p/prj.subrepo/ case
1990 			changeURL = "http://code.google.com/p/%s/source/detail?r=%s&repo=%s" % (m.group(6), changeURL, m.group(7)[1:])
1991 		elif m.group(4): # code.google.com/p/prj/ case
1992 			changeURL = "http://code.google.com/p/%s/source/detail?r=%s" % (m.group(6), changeURL)
1993 		else:
1994 			print >>sys.stderr, "URL: ", url
1995 	else:
1996 		print >>sys.stderr, "URL: ", url
1997 	pmsg = "*** Submitted as " + changeURL + " ***\n\n" + message
1998 
1999 	# When posting, move reviewers to CC line,
2000 	# so that the issue stops showing up in their "My Issues" page.
2001 	PostMessage(ui, cl.name, pmsg, reviewers="", cc=JoinComma(cl.reviewer+cl.cc))
2002 
2003 	if not cl.copied_from:
2004 		EditDesc(cl.name, closed=True, private=cl.private)
2005 	cl.Delete(ui, repo)
2006 
2007 	c = repo[None]
2008 	if c.branch() == releaseBranch and not c.modified() and not c.added() and not c.removed():
2009 		ui.write("switching from %s to default branch.\n" % releaseBranch)
2010 		err = hg_clean(repo, "default")
2011 		if err:
2012 			return err
2013 	return None
2014 
2015 #######################################################################
2016 # hg sync
2017 
2018 @hgcommand
2019 def sync(ui, repo, **opts):
2020 	"""synchronize with remote repository
2021 
2022 	Incorporates recent changes from the remote repository
2023 	into the local repository.
2024 	"""
2025 	if codereview_disabled:
2026 		return codereview_disabled
2027 
2028 	if not opts["local"]:
2029 		err = hg_pull(ui, repo, update=True)
2030 		if err:
2031 			return err
2032 	sync_changes(ui, repo)
2033 
2034 def sync_changes(ui, repo):
2035 	# Look through recent change log descriptions to find
2036 	# potential references to http://.*/our-CL-number.
2037 	# Double-check them by looking at the Rietveld log.
2038 	for rev in hg_log(ui, repo, limit=100, template="{node}\n").split():
2039 		desc = repo[rev].description().strip()
2040 		for clname in re.findall('(?m)^http://(?:[^\n]+)/([0-9]+)$', desc):
2041 			if IsLocalCL(ui, repo, clname) and IsRietveldSubmitted(ui, clname, repo[rev].hex()):
2042 				ui.warn("CL %s submitted as %s; closing\n" % (clname, repo[rev]))
2043 				cl, err = LoadCL(ui, repo, clname, web=False)
2044 				if err != "":
2045 					ui.warn("loading CL %s: %s\n" % (clname, err))
2046 					continue
2047 				if not cl.copied_from:
2048 					EditDesc(cl.name, closed=True, private=cl.private)
2049 				cl.Delete(ui, repo)
2050 
2051 	# Remove files that are not modified from the CLs in which they appear.
2052 	all = LoadAllCL(ui, repo, web=False)
2053 	changed = ChangedFiles(ui, repo, [])
2054 	for cl in all.values():
2055 		extra = Sub(cl.files, changed)
2056 		if extra:
2057 			ui.warn("Removing unmodified files from CL %s:\n" % (cl.name,))
2058 			for f in extra:
2059 				ui.warn("\t%s\n" % (f,))
2060 			cl.files = Sub(cl.files, extra)
2061 			cl.Flush(ui, repo)
2062 		if not cl.files:
2063 			if not cl.copied_from:
2064 				ui.warn("CL %s has no files; delete (abandon) with hg change -d %s\n" % (cl.name, cl.name))
2065 			else:
2066 				ui.warn("CL %s has no files; delete locally with hg change -D %s\n" % (cl.name, cl.name))
2067 	return
2068 
2069 #######################################################################
2070 # hg upload
2071 
2072 @hgcommand
2073 def upload(ui, repo, name, **opts):
2074 	"""upload diffs to the code review server
2075 
2076 	Uploads the current modifications for a given change to the server.
2077 	"""
2078 	if codereview_disabled:
2079 		return codereview_disabled
2080 
2081 	repo.ui.quiet = True
2082 	cl, err = LoadCL(ui, repo, name, web=True)
2083 	if err != "":
2084 		return err
2085 	if not cl.local:
2086 		return "cannot upload non-local change"
2087 	cl.Upload(ui, repo)
2088 	print "%s%s\n" % (server_url_base, cl.name)
2089 	return
2090 
2091 #######################################################################
2092 # Table of commands, supplied to Mercurial for installation.
2093 
2094 review_opts = [
2095 	('r', 'reviewer', '', 'add reviewer'),
2096 	('', 'cc', '', 'add cc'),
2097 	('', 'tbr', '', 'add future reviewer'),
2098 	('m', 'message', '', 'change description (for new change)'),
2099 ]
2100 
2101 cmdtable = {
2102 	# The ^ means to show this command in the help text that
2103 	# is printed when running hg with no arguments.
2104 	"^change": (
2105 		change,
2106 		[
2107 			('d', 'delete', None, 'delete existing change list'),
2108 			('D', 'deletelocal', None, 'delete locally, but do not change CL on server'),
2109 			('i', 'stdin', None, 'read change list from standard input'),
2110 			('o', 'stdout', None, 'print change list to standard output'),
2111 			('p', 'pending', None, 'print pending summary to standard output'),
2112 		],
2113 		"[-d | -D] [-i] [-o] change# or FILE ..."
2114 	),
2115 	"^clpatch": (
2116 		clpatch,
2117 		[
2118 			('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
2119 			('', 'no_incoming', None, 'disable check for incoming changes'),
2120 		],
2121 		"change#"
2122 	),
2123 	# Would prefer to call this codereview-login, but then
2124 	# hg help codereview prints the help for this command
2125 	# instead of the help for the extension.
2126 	"code-login": (
2127 		code_login,
2128 		[],
2129 		"",
2130 	),
2131 	"^download": (
2132 		download,
2133 		[],
2134 		"change#"
2135 	),
2136 	"^file": (
2137 		file,
2138 		[
2139 			('d', 'delete', None, 'delete files from change list (but not repository)'),
2140 		],
2141 		"[-d] change# FILE ..."
2142 	),
2143 	"^gofmt": (
2144 		gofmt,
2145 		[
2146 			('l', 'list', None, 'list files that would change, but do not edit them'),
2147 		],
2148 		"FILE ..."
2149 	),
2150 	"^pending|p": (
2151 		pending,
2152 		[
2153 			('s', 'short', False, 'show short result form'),
2154 			('', 'quick', False, 'do not consult codereview server'),
2155 		],
2156 		"[FILE ...]"
2157 	),
2158 	"^ps": (
2159 		ps,
2160 		[],
2161 		"[FILE ...]"
2162 	),
2163 	"^pq": (
2164 		pq,
2165 		[],
2166 		"[FILE ...]"
2167 	),
2168 	"^mail": (
2169 		mail,
2170 		review_opts + [
2171 		] + hg_commands.walkopts,
2172 		"[-r reviewer] [--cc cc] [change# | file ...]"
2173 	),
2174 	"^release-apply": (
2175 		release_apply,
2176 		[
2177 			('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
2178 			('', 'no_incoming', None, 'disable check for incoming changes'),
2179 		],
2180 		"change#"
2181 	),
2182 	# TODO: release-start, release-tag, weekly-tag
2183 	"^submit": (
2184 		submit,
2185 		review_opts + [
2186 			('', 'no_incoming', None, 'disable initial incoming check (for testing)'),
2187 		] + hg_commands.walkopts + hg_commands.commitopts + hg_commands.commitopts2,
2188 		"[-r reviewer] [--cc cc] [change# | file ...]"
2189 	),
2190 	"^sync": (
2191 		sync,
2192 		[
2193 			('', 'local', None, 'do not pull changes from remote repository')
2194 		],
2195 		"[--local]",
2196 	),
2197 	"^undo": (
2198 		undo,
2199 		[
2200 			('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
2201 			('', 'no_incoming', None, 'disable check for incoming changes'),
2202 		],
2203 		"change#"
2204 	),
2205 	"^upload": (
2206 		upload,
2207 		[],
2208 		"change#"
2209 	),
2210 }
2211 
2212 #######################################################################
2213 # Mercurial extension initialization
2214 
2215 def norollback(*pats, **opts):
2216 	"""(disabled when using this extension)"""
2217 	raise hg_util.Abort("codereview extension enabled; use undo instead of rollback")
2218 
2219 codereview_init = False
2220 
2221 def reposetup(ui, repo):
2222 	global codereview_disabled
2223 	global defaultcc
2224 	
2225 	# reposetup gets called both for the local repository
2226 	# and also for any repository we are pulling or pushing to.
2227 	# Only initialize the first time.
2228 	global codereview_init
2229 	if codereview_init:
2230 		return
2231 	codereview_init = True
2232 
2233 	# Read repository-specific options from lib/codereview/codereview.cfg or codereview.cfg.
2234 	root = ''
2235 	try:
2236 		root = repo.root
2237 	except:
2238 		# Yes, repo might not have root; see issue 959.
2239 		codereview_disabled = 'codereview disabled: repository has no root'
2240 		return
2241 	
2242 	repo_config_path = ''
2243 	p1 = root + '/lib/codereview/codereview.cfg'
2244 	p2 = root + '/codereview.cfg'
2245 	if os.access(p1, os.F_OK):
2246 		repo_config_path = p1
2247 	else:
2248 		repo_config_path = p2
2249 	try:
2250 		f = open(repo_config_path)
2251 		for line in f:
2252 			if line.startswith('defaultcc:'):
2253 				defaultcc = SplitCommaSpace(line[len('defaultcc:'):])
2254 			if line.startswith('contributors:'):
2255 				global contributorsURL
2256 				contributorsURL = line[len('contributors:'):].strip()
2257 	except:
2258 		codereview_disabled = 'codereview disabled: cannot open ' + repo_config_path
2259 		return
2260 
2261 	remote = ui.config("paths", "default", "")
2262 	if remote.find("://") < 0:
2263 		raise hg_util.Abort("codereview: default path '%s' is not a URL" % (remote,))
2264 
2265 	InstallMatch(ui, repo)
2266 	RietveldSetup(ui, repo)
2267 
2268 	# Disable the Mercurial commands that might change the repository.
2269 	# Only commands in this extension are supposed to do that.
2270 	ui.setconfig("hooks", "precommit.codereview", precommithook)
2271 
2272 	# Rollback removes an existing commit.  Don't do that either.
2273 	global real_rollback
2274 	real_rollback = repo.rollback
2275 	repo.rollback = norollback
2276 	
2277 
2278 #######################################################################
2279 # Wrappers around upload.py for interacting with Rietveld
2280 
2281 from HTMLParser import HTMLParser
2282 
2283 # HTML form parser
2284 class FormParser(HTMLParser):
2285 	def __init__(self):
2286 		self.map = {}
2287 		self.curtag = None
2288 		self.curdata = None
2289 		HTMLParser.__init__(self)
2290 	def handle_starttag(self, tag, attrs):
2291 		if tag == "input":
2292 			key = None
2293 			value = ''
2294 			for a in attrs:
2295 				if a[0] == 'name':
2296 					key = a[1]
2297 				if a[0] == 'value':
2298 					value = a[1]
2299 			if key is not None:
2300 				self.map[key] = value
2301 		if tag == "textarea":
2302 			key = None
2303 			for a in attrs:
2304 				if a[0] == 'name':
2305 					key = a[1]
2306 			if key is not None:
2307 				self.curtag = key
2308 				self.curdata = ''
2309 	def handle_endtag(self, tag):
2310 		if tag == "textarea" and self.curtag is not None:
2311 			self.map[self.curtag] = self.curdata
2312 			self.curtag = None
2313 			self.curdata = None
2314 	def handle_charref(self, name):
2315 		self.handle_data(unichr(int(name)))
2316 	def handle_entityref(self, name):
2317 		import htmlentitydefs
2318 		if name in htmlentitydefs.entitydefs:
2319 			self.handle_data(htmlentitydefs.entitydefs[name])
2320 		else:
2321 			self.handle_data("&" + name + ";")
2322 	def handle_data(self, data):
2323 		if self.curdata is not None:
2324 			self.curdata += data
2325 
2326 def JSONGet(ui, path):
2327 	try:
2328 		data = MySend(path, force_auth=False)
2329 		typecheck(data, str)
2330 		d = fix_json(json.loads(data))
2331 	except:
2332 		ui.warn("JSONGet %s: %s\n" % (path, ExceptionDetail()))
2333 		return None
2334 	return d
2335 
2336 # Clean up json parser output to match our expectations:
2337 #   * all strings are UTF-8-encoded str, not unicode.
2338 #   * missing fields are missing, not None,
2339 #     so that d.get("foo", defaultvalue) works.
2340 def fix_json(x):
2341 	if type(x) in [str, int, float, bool, type(None)]:
2342 		pass
2343 	elif type(x) is unicode:
2344 		x = x.encode("utf-8")
2345 	elif type(x) is list:
2346 		for i in range(len(x)):
2347 			x[i] = fix_json(x[i])
2348 	elif type(x) is dict:
2349 		todel = []
2350 		for k in x:
2351 			if x[k] is None:
2352 				todel.append(k)
2353 			else:
2354 				x[k] = fix_json(x[k])
2355 		for k in todel:
2356 			del x[k]
2357 	else:
2358 		raise hg_util.Abort("unknown type " + str(type(x)) + " in fix_json")
2359 	if type(x) is str:
2360 		x = x.replace('\r\n', '\n')
2361 	return x
2362 
2363 def IsRietveldSubmitted(ui, clname, hex):
2364 	dict = JSONGet(ui, "/api/" + clname + "?messages=true")
2365 	if dict is None:
2366 		return False
2367 	for msg in dict.get("messages", []):
2368 		text = msg.get("text", "")
2369 		m = re.match('\*\*\* Submitted as [^*]*?([0-9a-f]+) \*\*\*', text)
2370 		if m is not None and len(m.group(1)) >= 8 and hex.startswith(m.group(1)):
2371 			return True
2372 	return False
2373 
2374 def IsRietveldMailed(cl):
2375 	for msg in cl.dict.get("messages", []):
2376 		if msg.get("text", "").find("I'd like you to review this change") >= 0:
2377 			return True
2378 	return False
2379 
2380 def DownloadCL(ui, repo, clname):
2381 	set_status("downloading CL " + clname)
2382 	cl, err = LoadCL(ui, repo, clname, web=True)
2383 	if err != "":
2384 		return None, None, None, "error loading CL %s: %s" % (clname, err)
2385 
2386 	# Find most recent diff
2387 	diffs = cl.dict.get("patchsets", [])
2388 	if not diffs:
2389 		return None, None, None, "CL has no patch sets"
2390 	patchid = diffs[-1]
2391 
2392 	patchset = JSONGet(ui, "/api/" + clname + "/" + str(patchid))
2393 	if patchset is None:
2394 		return None, None, None, "error loading CL patchset %s/%d" % (clname, patchid)
2395 	if patchset.get("patchset", 0) != patchid:
2396 		return None, None, None, "malformed patchset information"
2397 	
2398 	vers = ""
2399 	msg = patchset.get("message", "").split()
2400 	if len(msg) >= 3 and msg[0] == "diff" and msg[1] == "-r":
2401 		vers = msg[2]
2402 	diff = "/download/issue" + clname + "_" + str(patchid) + ".diff"
2403 
2404 	diffdata = MySend(diff, force_auth=False)
2405 	
2406 	# Print warning if email is not in CONTRIBUTORS file.
2407 	email = cl.dict.get("owner_email", "")
2408 	if not email:
2409 		return None, None, None, "cannot find owner for %s" % (clname)
2410 	him = FindContributor(ui, repo, email)
2411 	me = FindContributor(ui, repo, None)
2412 	if him == me:
2413 		cl.mailed = IsRietveldMailed(cl)
2414 	else:
2415 		cl.copied_from = email
2416 
2417 	return cl, vers, diffdata, ""
2418 
2419 def MySend(request_path, payload=None,
2420 		content_type="application/octet-stream",
2421 		timeout=None, force_auth=True,
2422 		**kwargs):
2423 	"""Run MySend1 maybe twice, because Rietveld is unreliable."""
2424 	try:
2425 		return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
2426 	except Exception, e:
2427 		if type(e) != urllib2.HTTPError or e.code != 500:	# only retry on HTTP 500 error
2428 			raise
2429 		print >>sys.stderr, "Loading "+request_path+": "+ExceptionDetail()+"; trying again in 2 seconds."
2430 		time.sleep(2)
2431 		return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
2432 
2433 # Like upload.py Send but only authenticates when the
2434 # redirect is to www.google.com/accounts.  This keeps
2435 # unnecessary redirects from happening during testing.
2436 def MySend1(request_path, payload=None,
2437 				content_type="application/octet-stream",
2438 				timeout=None, force_auth=True,
2439 				**kwargs):
2440 	"""Sends an RPC and returns the response.
2441 
2442 	Args:
2443 		request_path: The path to send the request to, eg /api/appversion/create.
2444 		payload: The body of the request, or None to send an empty request.
2445 		content_type: The Content-Type header to use.
2446 		timeout: timeout in seconds; default None i.e. no timeout.
2447 			(Note: for large requests on OS X, the timeout doesn't work right.)
2448 		kwargs: Any keyword arguments are converted into query string parameters.
2449 
2450 	Returns:
2451 		The response body, as a string.
2452 	"""
2453 	# TODO: Don't require authentication.  Let the server say
2454 	# whether it is necessary.
2455 	global rpc
2456 	if rpc == None:
2457 		rpc = GetRpcServer(upload_options)
2458 	self = rpc
2459 	if not self.authenticated and force_auth:
2460 		self._Authenticate()
2461 	if request_path is None:
2462 		return
2463 
2464 	old_timeout = socket.getdefaulttimeout()
2465 	socket.setdefaulttimeout(timeout)
2466 	try:
2467 		tries = 0
2468 		while True:
2469 			tries += 1
2470 			args = dict(kwargs)
2471 			url = "http://%s%s" % (self.host, request_path)
2472 			if args:
2473 				url += "?" + urllib.urlencode(args)
2474 			req = self._CreateRequest(url=url, data=payload)
2475 			req.add_header("Content-Type", content_type)
2476 			try:
2477 				f = self.opener.open(req)
2478 				response = f.read()
2479 				f.close()
2480 				# Translate \r\n into \n, because Rietveld doesn't.
2481 				response = response.replace('\r\n', '\n')
2482 				# who knows what urllib will give us
2483 				if type(response) == unicode:
2484 					response = response.encode("utf-8")
2485 				typecheck(response, str)
2486 				return response
2487 			except urllib2.HTTPError, e:
2488 				if tries > 3:
2489 					raise
2490 				elif e.code == 401:
2491 					self._Authenticate()
2492 				elif e.code == 302:
2493 					loc = e.info()["location"]
2494 					if not loc.startswith('https://www.google.com/a') or loc.find('/ServiceLogin') < 0:
2495 						return ''
2496 					self._Authenticate()
2497 				else:
2498 					raise
2499 	finally:
2500 		socket.setdefaulttimeout(old_timeout)
2501 
2502 def GetForm(url):
2503 	f = FormParser()
2504 	f.feed(ustr(MySend(url)))	# f.feed wants unicode
2505 	f.close()
2506 	# convert back to utf-8 to restore sanity
2507 	m = {}
2508 	for k,v in f.map.items():
2509 		m[k.encode("utf-8")] = v.replace("\r\n", "\n").encode("utf-8")
2510 	return m
2511 
2512 def EditDesc(issue, subject=None, desc=None, reviewers=None, cc=None, closed=False, private=False):
2513 	set_status("uploading change to description")
2514 	form_fields = GetForm("/" + issue + "/edit")
2515 	if subject is not None:
2516 		form_fields['subject'] = subject
2517 	if desc is not None:
2518 		form_fields['description'] = desc
2519 	if reviewers is not None:
2520 		form_fields['reviewers'] = reviewers
2521 	if cc is not None:
2522 		form_fields['cc'] = cc
2523 	if closed:
2524 		form_fields['closed'] = "checked"
2525 	if private:
2526 		form_fields['private'] = "checked"
2527 	ctype, body = EncodeMultipartFormData(form_fields.items(), [])
2528 	response = MySend("/" + issue + "/edit", body, content_type=ctype)
2529 	if response != "":
2530 		print >>sys.stderr, "Error editing description:\n" + "Sent form: \n", form_fields, "\n", response
2531 		sys.exit(2)
2532 
2533 def PostMessage(ui, issue, message, reviewers=None, cc=None, send_mail=True, subject=None):
2534 	set_status("uploading message")
2535 	form_fields = GetForm("/" + issue + "/publish")
2536 	if reviewers is not None:
2537 		form_fields['reviewers'] = reviewers
2538 	if cc is not None:
2539 		form_fields['cc'] = cc
2540 	if send_mail:
2541 		form_fields['send_mail'] = "checked"
2542 	else:
2543 		del form_fields['send_mail']
2544 	if subject is not None:
2545 		form_fields['subject'] = subject
2546 	form_fields['message'] = message
2547 	
2548 	form_fields['message_only'] = '1'	# Don't include draft comments
2549 	if reviewers is not None or cc is not None:
2550 		form_fields['message_only'] = ''	# Must set '' in order to override cc/reviewer
2551 	ctype = "applications/x-www-form-urlencoded"
2552 	body = urllib.urlencode(form_fields)
2553 	response = MySend("/" + issue + "/publish", body, content_type=ctype)
2554 	if response != "":
2555 		print response
2556 		sys.exit(2)
2557 
2558 class opt(object):
2559 	pass
2560 
2561 def RietveldSetup(ui, repo):
2562 	global force_google_account
2563 	global rpc
2564 	global server
2565 	global server_url_base
2566 	global upload_options
2567 	global verbosity
2568 
2569 	if not ui.verbose:
2570 		verbosity = 0
2571 
2572 	# Config options.
2573 	x = ui.config("codereview", "server")
2574 	if x is not None:
2575 		server = x
2576 
2577 	# TODO(rsc): Take from ui.username?
2578 	email = None
2579 	x = ui.config("codereview", "email")
2580 	if x is not None:
2581 		email = x
2582 
2583 	server_url_base = "http://" + server + "/"
2584 
2585 	testing = ui.config("codereview", "testing")
2586 	force_google_account = ui.configbool("codereview", "force_google_account", False)
2587 
2588 	upload_options = opt()
2589 	upload_options.email = email
2590 	upload_options.host = None
2591 	upload_options.verbose = 0
2592 	upload_options.description = None
2593 	upload_options.description_file = None
2594 	upload_options.reviewers = None
2595 	upload_options.cc = None
2596 	upload_options.message = None
2597 	upload_options.issue = None
2598 	upload_options.download_base = False
2599 	upload_options.revision = None
2600 	upload_options.send_mail = False
2601 	upload_options.vcs = None
2602 	upload_options.server = server
2603 	upload_options.save_cookies = True
2604 
2605 	if testing:
2606 		upload_options.save_cookies = False
2607 		upload_options.email = "test@example.com"
2608 
2609 	rpc = None
2610 	
2611 	global releaseBranch
2612 	tags = repo.branchtags().keys()
2613 	if 'release-branch.go10' in tags:
2614 		# NOTE(rsc): This tags.sort is going to get the wrong
2615 		# answer when comparing release-branch.go9 with
2616 		# release-branch.go10.  It will be a while before we care.
2617 		raise hg_util.Abort('tags.sort needs to be fixed for release-branch.go10')
2618 	tags.sort()
2619 	for t in tags:
2620 		if t.startswith('release-branch.go'):
2621 			releaseBranch = t			
2622 
2623 #######################################################################
2624 # http://codereview.appspot.com/static/upload.py, heavily edited.
2625 
2626 #!/usr/bin/env python
2627 #
2628 # Copyright 2007 Google Inc.
2629 #
2630 # Licensed under the Apache License, Version 2.0 (the "License");
2631 # you may not use this file except in compliance with the License.
2632 # You may obtain a copy of the License at
2633 #
2634 #	http://www.apache.org/licenses/LICENSE-2.0
2635 #
2636 # Unless required by applicable law or agreed to in writing, software
2637 # distributed under the License is distributed on an "AS IS" BASIS,
2638 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2639 # See the License for the specific language governing permissions and
2640 # limitations under the License.
2641 
2642 """Tool for uploading diffs from a version control system to the codereview app.
2643 
2644 Usage summary: upload.py [options] [-- diff_options]
2645 
2646 Diff options are passed to the diff command of the underlying system.
2647 
2648 Supported version control systems:
2649 	Git
2650 	Mercurial
2651 	Subversion
2652 
2653 It is important for Git/Mercurial users to specify a tree/node/branch to diff
2654 against by using the '--rev' option.
2655 """
2656 # This code is derived from appcfg.py in the App Engine SDK (open source),
2657 # and from ASPN recipe #146306.
2658 
2659 import cookielib
2660 import getpass
2661 import logging
2662 import mimetypes
2663 import optparse
2664 import os
2665 import re
2666 import socket
2667 import subprocess
2668 import sys
2669 import urllib
2670 import urllib2
2671 import urlparse
2672 
2673 # The md5 module was deprecated in Python 2.5.
2674 try:
2675 	from hashlib import md5
2676 except ImportError:
2677 	from md5 import md5
2678 
2679 try:
2680 	import readline
2681 except ImportError:
2682 	pass
2683 
2684 # The logging verbosity:
2685 #  0: Errors only.
2686 #  1: Status messages.
2687 #  2: Info logs.
2688 #  3: Debug logs.
2689 verbosity = 1
2690 
2691 # Max size of patch or base file.
2692 MAX_UPLOAD_SIZE = 900 * 1024
2693 
2694 # whitelist for non-binary filetypes which do not start with "text/"
2695 # .mm (Objective-C) shows up as application/x-freemind on my Linux box.
2696 TEXT_MIMETYPES = [
2697 	'application/javascript',
2698 	'application/x-javascript',
2699 	'application/x-freemind'
2700 ]
2701 
2702 def GetEmail(prompt):
2703 	"""Prompts the user for their email address and returns it.
2704 
2705 	The last used email address is saved to a file and offered up as a suggestion
2706 	to the user. If the user presses enter without typing in anything the last
2707 	used email address is used. If the user enters a new address, it is saved
2708 	for next time we prompt.
2709 
2710 	"""
2711 	last_email_file_name = os.path.expanduser("~/.last_codereview_email_address")
2712 	last_email = ""
2713 	if os.path.exists(last_email_file_name):
2714 		try:
2715 			last_email_file = open(last_email_file_name, "r")
2716 			last_email = last_email_file.readline().strip("\n")
2717 			last_email_file.close()
2718 			prompt += " [%s]" % last_email
2719 		except IOError, e:
2720 			pass
2721 	email = raw_input(prompt + ": ").strip()
2722 	if email:
2723 		try:
2724 			last_email_file = open(last_email_file_name, "w")
2725 			last_email_file.write(email)
2726 			last_email_file.close()
2727 		except IOError, e:
2728 			pass
2729 	else:
2730 		email = last_email
2731 	return email
2732 
2733 
2734 def StatusUpdate(msg):
2735 	"""Print a status message to stdout.
2736 
2737 	If 'verbosity' is greater than 0, print the message.
2738 
2739 	Args:
2740 		msg: The string to print.
2741 	"""
2742 	if verbosity > 0:
2743 		print msg
2744 
2745 
2746 def ErrorExit(msg):
2747 	"""Print an error message to stderr and exit."""
2748 	print >>sys.stderr, msg
2749 	sys.exit(1)
2750 
2751 
2752 class ClientLoginError(urllib2.HTTPError):
2753 	"""Raised to indicate there was an error authenticating with ClientLogin."""
2754 
2755 	def __init__(self, url, code, msg, headers, args):
2756 		urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
2757 		self.args = args
2758 		self.reason = args["Error"]
2759 
2760 
2761 class AbstractRpcServer(object):
2762 	"""Provides a common interface for a simple RPC server."""
2763 
2764 	def __init__(self, host, auth_function, host_override=None, extra_headers={}, save_cookies=False):
2765 		"""Creates a new HttpRpcServer.
2766 
2767 		Args:
2768 			host: The host to send requests to.
2769 			auth_function: A function that takes no arguments and returns an
2770 				(email, password) tuple when called. Will be called if authentication
2771 				is required.
2772 			host_override: The host header to send to the server (defaults to host).
2773 			extra_headers: A dict of extra headers to append to every request.
2774 			save_cookies: If True, save the authentication cookies to local disk.
2775 				If False, use an in-memory cookiejar instead.  Subclasses must
2776 				implement this functionality.  Defaults to False.
2777 		"""
2778 		self.host = host
2779 		self.host_override = host_override
2780 		self.auth_function = auth_function
2781 		self.authenticated = False
2782 		self.extra_headers = extra_headers
2783 		self.save_cookies = save_cookies
2784 		self.opener = self._GetOpener()
2785 		if self.host_override:
2786 			logging.info("Server: %s; Host: %s", self.host, self.host_override)
2787 		else:
2788 			logging.info("Server: %s", self.host)
2789 
2790 	def _GetOpener(self):
2791 		"""Returns an OpenerDirector for making HTTP requests.
2792 
2793 		Returns:
2794 			A urllib2.OpenerDirector object.
2795 		"""
2796 		raise NotImplementedError()
2797 
2798 	def _CreateRequest(self, url, data=None):
2799 		"""Creates a new urllib request."""
2800 		logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
2801 		req = urllib2.Request(url, data=data)
2802 		if self.host_override:
2803 			req.add_header("Host", self.host_override)
2804 		for key, value in self.extra_headers.iteritems():
2805 			req.add_header(key, value)
2806 		return req
2807 
2808 	def _GetAuthToken(self, email, password):
2809 		"""Uses ClientLogin to authenticate the user, returning an auth token.
2810 
2811 		Args:
2812 			email:    The user's email address
2813 			password: The user's password
2814 
2815 		Raises:
2816 			ClientLoginError: If there was an error authenticating with ClientLogin.
2817 			HTTPError: If there was some other form of HTTP error.
2818 
2819 		Returns:
2820 			The authentication token returned by ClientLogin.
2821 		"""
2822 		account_type = "GOOGLE"
2823 		if self.host.endswith(".google.com") and not force_google_account:
2824 			# Needed for use inside Google.
2825 			account_type = "HOSTED"
2826 		req = self._CreateRequest(
2827 				url="https://www.google.com/accounts/ClientLogin",
2828 				data=urllib.urlencode({
2829 						"Email": email,
2830 						"Passwd": password,
2831 						"service": "ah",
2832 						"source": "rietveld-codereview-upload",
2833 						"accountType": account_type,
2834 				}),
2835 		)
2836 		try:
2837 			response = self.opener.open(req)
2838 			response_body = response.read()
2839 			response_dict = dict(x.split("=") for x in response_body.split("\n") if x)
2840 			return response_dict["Auth"]
2841 		except urllib2.HTTPError, e:
2842 			if e.code == 403:
2843 				body = e.read()
2844 				response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
2845 				raise ClientLoginError(req.get_full_url(), e.code, e.msg, e.headers, response_dict)
2846 			else:
2847 				raise
2848 
2849 	def _GetAuthCookie(self, auth_token):
2850 		"""Fetches authentication cookies for an authentication token.
2851 
2852 		Args:
2853 			auth_token: The authentication token returned by ClientLogin.
2854 
2855 		Raises:
2856 			HTTPError: If there was an error fetching the authentication cookies.
2857 		"""
2858 		# This is a dummy value to allow us to identify when we're successful.
2859 		continue_location = "http://localhost/"
2860 		args = {"continue": continue_location, "auth": auth_token}
2861 		req = self._CreateRequest("http://%s/_ah/login?%s" % (self.host, urllib.urlencode(args)))
2862 		try:
2863 			response = self.opener.open(req)
2864 		except urllib2.HTTPError, e:
2865 			response = e
2866 		if (response.code != 302 or
2867 				response.info()["location"] != continue_location):
2868 			raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg, response.headers, response.fp)
2869 		self.authenticated = True
2870 
2871 	def _Authenticate(self):
2872 		"""Authenticates the user.
2873 
2874 		The authentication process works as follows:
2875 		1) We get a username and password from the user
2876 		2) We use ClientLogin to obtain an AUTH token for the user
2877 				(see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
2878 		3) We pass the auth token to /_ah/login on the server to obtain an
2879 				authentication cookie. If login was successful, it tries to redirect
2880 				us to the URL we provided.
2881 
2882 		If we attempt to access the upload API without first obtaining an
2883 		authentication cookie, it returns a 401 response (or a 302) and
2884 		directs us to authenticate ourselves with ClientLogin.
2885 		"""
2886 		for i in range(3):
2887 			credentials = self.auth_function()
2888 			try:
2889 				auth_token = self._GetAuthToken(credentials[0], credentials[1])
2890 			except ClientLoginError, e:
2891 				if e.reason == "BadAuthentication":
2892 					print >>sys.stderr, "Invalid username or password."
2893 					continue
2894 				if e.reason == "CaptchaRequired":
2895 					print >>sys.stderr, (
2896 						"Please go to\n"
2897 						"https://www.google.com/accounts/DisplayUnlockCaptcha\n"
2898 						"and verify you are a human.  Then try again.")
2899 					break
2900 				if e.reason == "NotVerified":
2901 					print >>sys.stderr, "Account not verified."
2902 					break
2903 				if e.reason == "TermsNotAgreed":
2904 					print >>sys.stderr, "User has not agreed to TOS."
2905 					break
2906 				if e.reason == "AccountDeleted":
2907 					print >>sys.stderr, "The user account has been deleted."
2908 					break
2909 				if e.reason == "AccountDisabled":
2910 					print >>sys.stderr, "The user account has been disabled."
2911 					break
2912 				if e.reason == "ServiceDisabled":
2913 					print >>sys.stderr, "The user's access to the service has been disabled."
2914 					break
2915 				if e.reason == "ServiceUnavailable":
2916 					print >>sys.stderr, "The service is not available; try again later."
2917 					break
2918 				raise
2919 			self._GetAuthCookie(auth_token)
2920 			return
2921 
2922 	def Send(self, request_path, payload=None,
2923 					content_type="application/octet-stream",
2924 					timeout=None,
2925 					**kwargs):
2926 		"""Sends an RPC and returns the response.
2927 
2928 		Args:
2929 			request_path: The path to send the request to, eg /api/appversion/create.
2930 			payload: The body of the request, or None to send an empty request.
2931 			content_type: The Content-Type header to use.
2932 			timeout: timeout in seconds; default None i.e. no timeout.
2933 				(Note: for large requests on OS X, the timeout doesn't work right.)
2934 			kwargs: Any keyword arguments are converted into query string parameters.
2935 
2936 		Returns:
2937 			The response body, as a string.
2938 		"""
2939 		# TODO: Don't require authentication.  Let the server say
2940 		# whether it is necessary.
2941 		if not self.authenticated:
2942 			self._Authenticate()
2943 
2944 		old_timeout = socket.getdefaulttimeout()
2945 		socket.setdefaulttimeout(timeout)
2946 		try:
2947 			tries = 0
2948 			while True:
2949 				tries += 1
2950 				args = dict(kwargs)
2951 				url = "http://%s%s" % (self.host, request_path)
2952 				if args:
2953 					url += "?" + urllib.urlencode(args)
2954 				req = self._CreateRequest(url=url, data=payload)
2955 				req.add_header("Content-Type", content_type)
2956 				try:
2957 					f = self.opener.open(req)
2958 					response = f.read()
2959 					f.close()
2960 					return response
2961 				except urllib2.HTTPError, e:
2962 					if tries > 3:
2963 						raise
2964 					elif e.code == 401 or e.code == 302:
2965 						self._Authenticate()
2966 					else:
2967 						raise
2968 		finally:
2969 			socket.setdefaulttimeout(old_timeout)
2970 
2971 
2972 class HttpRpcServer(AbstractRpcServer):
2973 	"""Provides a simplified RPC-style interface for HTTP requests."""
2974 
2975 	def _Authenticate(self):
2976 		"""Save the cookie jar after authentication."""
2977 		super(HttpRpcServer, self)._Authenticate()
2978 		if self.save_cookies:
2979 			StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)
2980 			self.cookie_jar.save()
2981 
2982 	def _GetOpener(self):
2983 		"""Returns an OpenerDirector that supports cookies and ignores redirects.
2984 
2985 		Returns:
2986 			A urllib2.OpenerDirector object.
2987 		"""
2988 		opener = urllib2.OpenerDirector()
2989 		opener.add_handler(urllib2.ProxyHandler())
2990 		opener.add_handler(urllib2.UnknownHandler())
2991 		opener.add_handler(urllib2.HTTPHandler())
2992 		opener.add_handler(urllib2.HTTPDefaultErrorHandler())
2993 		opener.add_handler(urllib2.HTTPSHandler())
2994 		opener.add_handler(urllib2.HTTPErrorProcessor())
2995 		if self.save_cookies:
2996 			self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies_" + server)
2997 			self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
2998 			if os.path.exists(self.cookie_file):
2999 				try:
3000 					self.cookie_jar.load()
3001 					self.authenticated = True
3002 					StatusUpdate("Loaded authentication cookies from %s" % self.cookie_file)
3003 				except (cookielib.LoadError, IOError):
3004 					# Failed to load cookies - just ignore them.
3005 					pass
3006 			else:
3007 				# Create an empty cookie file with mode 600
3008 				fd = os.open(self.cookie_file, os.O_CREAT, 0600)
3009 				os.close(fd)
3010 			# Always chmod the cookie file
3011 			os.chmod(self.cookie_file, 0600)
3012 		else:
3013 			# Don't save cookies across runs of update.py.
3014 			self.cookie_jar = cookielib.CookieJar()
3015 		opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
3016 		return opener
3017 
3018 
3019 def GetRpcServer(options):
3020 	"""Returns an instance of an AbstractRpcServer.
3021 
3022 	Returns:
3023 		A new AbstractRpcServer, on which RPC calls can be made.
3024 	"""
3025 
3026 	rpc_server_class = HttpRpcServer
3027 
3028 	def GetUserCredentials():
3029 		"""Prompts the user for a username and password."""
3030 		# Disable status prints so they don't obscure the password prompt.
3031 		global global_status
3032 		st = global_status
3033 		global_status = None
3034 
3035 		email = options.email
3036 		if email is None:
3037 			email = GetEmail("Email (login for uploading to %s)" % options.server)
3038 		password = getpass.getpass("Password for %s: " % email)
3039 
3040 		# Put status back.
3041 		global_status = st
3042 		return (email, password)
3043 
3044 	# If this is the dev_appserver, use fake authentication.
3045 	host = (options.host or options.server).lower()
3046 	if host == "localhost" or host.startswith("localhost:"):
3047 		email = options.email
3048 		if email is None:
3049 			email = "test@example.com"
3050 			logging.info("Using debug user %s.  Override with --email" % email)
3051 		server = rpc_server_class(
3052 				options.server,
3053 				lambda: (email, "password"),
3054 				host_override=options.host,
3055 				extra_headers={"Cookie": 'dev_appserver_login="%s:False"' % email},
3056 				save_cookies=options.save_cookies)
3057 		# Don't try to talk to ClientLogin.
3058 		server.authenticated = True
3059 		return server
3060 
3061 	return rpc_server_class(options.server, GetUserCredentials,
3062 		host_override=options.host, save_cookies=options.save_cookies)
3063 
3064 
3065 def EncodeMultipartFormData(fields, files):
3066 	"""Encode form fields for multipart/form-data.
3067 
3068 	Args:
3069 		fields: A sequence of (name, value) elements for regular form fields.
3070 		files: A sequence of (name, filename, value) elements for data to be
3071 					uploaded as files.
3072 	Returns:
3073 		(content_type, body) ready for httplib.HTTP instance.
3074 
3075 	Source:
3076 		http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
3077 	"""
3078 	BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
3079 	CRLF = '\r\n'
3080 	lines = []
3081 	for (key, value) in fields:
3082 		typecheck(key, str)
3083 		typecheck(value, str)
3084 		lines.append('--' + BOUNDARY)
3085 		lines.append('Content-Disposition: form-data; name="%s"' % key)
3086 		lines.append('')
3087 		lines.append(value)
3088 	for (key, filename, value) in files:
3089 		typecheck(key, str)
3090 		typecheck(filename, str)
3091 		typecheck(value, str)
3092 		lines.append('--' + BOUNDARY)
3093 		lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
3094 		lines.append('Content-Type: %s' % GetContentType(filename))
3095 		lines.append('')
3096 		lines.append(value)
3097 	lines.append('--' + BOUNDARY + '--')
3098 	lines.append('')
3099 	body = CRLF.join(lines)
3100 	content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
3101 	return content_type, body
3102 
3103 
3104 def GetContentType(filename):
3105 	"""Helper to guess the content-type from the filename."""
3106 	return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
3107 
3108 
3109 # Use a shell for subcommands on Windows to get a PATH search.
3110 use_shell = sys.platform.startswith("win")
3111 
3112 def RunShellWithReturnCode(command, print_output=False,
3113 		universal_newlines=True, env=os.environ):
3114 	"""Executes a command and returns the output from stdout and the return code.
3115 
3116 	Args:
3117 		command: Command to execute.
3118 		print_output: If True, the output is printed to stdout.
3119 			If False, both stdout and stderr are ignored.
3120 		universal_newlines: Use universal_newlines flag (default: True).
3121 
3122 	Returns:
3123 		Tuple (output, return code)
3124 	"""
3125 	logging.info("Running %s", command)
3126 	p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
3127 		shell=use_shell, universal_newlines=universal_newlines, env=env)
3128 	if print_output:
3129 		output_array = []
3130 		while True:
3131 			line = p.stdout.readline()
3132 			if not line:
3133 				break
3134 			print line.strip("\n")
3135 			output_array.append(line)
3136 		output = "".join(output_array)
3137 	else:
3138 		output = p.stdout.read()
3139 	p.wait()
3140 	errout = p.stderr.read()
3141 	if print_output and errout:
3142 		print >>sys.stderr, errout
3143 	p.stdout.close()
3144 	p.stderr.close()
3145 	return output, p.returncode
3146 
3147 
3148 def RunShell(command, silent_ok=False, universal_newlines=True,
3149 		print_output=False, env=os.environ):
3150 	data, retcode = RunShellWithReturnCode(command, print_output, universal_newlines, env)
3151 	if retcode:
3152 		ErrorExit("Got error status from %s:\n%s" % (command, data))
3153 	if not silent_ok and not data:
3154 		ErrorExit("No output from %s" % command)
3155 	return data
3156 
3157 
3158 class VersionControlSystem(object):
3159 	"""Abstract base class providing an interface to the VCS."""
3160 
3161 	def __init__(self, options):
3162 		"""Constructor.
3163 
3164 		Args:
3165 			options: Command line options.
3166 		"""
3167 		self.options = options
3168 
3169 	def GenerateDiff(self, args):
3170 		"""Return the current diff as a string.
3171 
3172 		Args:
3173 			args: Extra arguments to pass to the diff command.
3174 		"""
3175 		raise NotImplementedError(
3176 				"abstract method -- subclass %s must override" % self.__class__)
3177 
3178 	def GetUnknownFiles(self):
3179 		"""Return a list of files unknown to the VCS."""
3180 		raise NotImplementedError(
3181 				"abstract method -- subclass %s must override" % self.__class__)
3182 
3183 	def CheckForUnknownFiles(self):
3184 		"""Show an "are you sure?" prompt if there are unknown files."""
3185 		unknown_files = self.GetUnknownFiles()
3186 		if unknown_files:
3187 			print "The following files are not added to version control:"
3188 			for line in unknown_files:
3189 				print line
3190 			prompt = "Are you sure to continue?(y/N) "
3191 			answer = raw_input(prompt).strip()
3192 			if answer != "y":
3193 				ErrorExit("User aborted")
3194 
3195 	def GetBaseFile(self, filename):
3196 		"""Get the content of the upstream version of a file.
3197 
3198 		Returns:
3199 			A tuple (base_content, new_content, is_binary, status)
3200 				base_content: The contents of the base file.
3201 				new_content: For text files, this is empty.  For binary files, this is
3202 					the contents of the new file, since the diff output won't contain
3203 					information to reconstruct the current file.
3204 				is_binary: True iff the file is binary.
3205 				status: The status of the file.
3206 		"""
3207 
3208 		raise NotImplementedError(
3209 				"abstract method -- subclass %s must override" % self.__class__)
3210 
3211 
3212 	def GetBaseFiles(self, diff):
3213 		"""Helper that calls GetBase file for each file in the patch.
3214 
3215 		Returns:
3216 			A dictionary that maps from filename to GetBaseFile's tuple.  Filenames
3217 			are retrieved based on lines that start with "Index:" or
3218 			"Property changes on:".
3219 		"""
3220 		files = {}
3221 		for line in diff.splitlines(True):
3222 			if line.startswith('Index:') or line.startswith('Property changes on:'):
3223 				unused, filename = line.split(':', 1)
3224 				# On Windows if a file has property changes its filename uses '\'
3225 				# instead of '/'.
3226 				filename = to_slash(filename.strip())
3227 				files[filename] = self.GetBaseFile(filename)
3228 		return files
3229 
3230 
3231 	def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options,
3232 											files):
3233 		"""Uploads the base files (and if necessary, the current ones as well)."""
3234 
3235 		def UploadFile(filename, file_id, content, is_binary, status, is_base):
3236 			"""Uploads a file to the server."""
3237 			set_status("uploading " + filename)
3238 			file_too_large = False
3239 			if is_base:
3240 				type = "base"
3241 			else:
3242 				type = "current"
3243 			if len(content) > MAX_UPLOAD_SIZE:
3244 				print ("Not uploading the %s file for %s because it's too large." %
3245 							(type, filename))
3246 				file_too_large = True
3247 				content = ""
3248 			checksum = md5(content).hexdigest()
3249 			if options.verbose > 0 and not file_too_large:
3250 				print "Uploading %s file for %s" % (type, filename)
3251 			url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id)
3252 			form_fields = [
3253 				("filename", filename),
3254 				("status", status),
3255 				("checksum", checksum),
3256 				("is_binary", str(is_binary)),
3257 				("is_current", str(not is_base)),
3258 			]
3259 			if file_too_large:
3260 				form_fields.append(("file_too_large", "1"))
3261 			if options.email:
3262 				form_fields.append(("user", options.email))
3263 			ctype, body = EncodeMultipartFormData(form_fields, [("data", filename, content)])
3264 			response_body = rpc_server.Send(url, body, content_type=ctype)
3265 			if not response_body.startswith("OK"):
3266 				StatusUpdate("  --> %s" % response_body)
3267 				sys.exit(1)
3268 
3269 		# Don't want to spawn too many threads, nor do we want to
3270 		# hit Rietveld too hard, or it will start serving 500 errors.
3271 		# When 8 works, it's no better than 4, and sometimes 8 is
3272 		# too many for Rietveld to handle.
3273 		MAX_PARALLEL_UPLOADS = 4
3274 
3275 		sema = threading.BoundedSemaphore(MAX_PARALLEL_UPLOADS)
3276 		upload_threads = []
3277 		finished_upload_threads = []
3278 		
3279 		class UploadFileThread(threading.Thread):
3280 			def __init__(self, args):
3281 				threading.Thread.__init__(self)
3282 				self.args = args
3283 			def run(self):
3284 				UploadFile(*self.args)
3285 				finished_upload_threads.append(self)
3286 				sema.release()
3287 
3288 		def StartUploadFile(*args):
3289 			sema.acquire()
3290 			while len(finished_upload_threads) > 0:
3291 				t = finished_upload_threads.pop()
3292 				upload_threads.remove(t)
3293 				t.join()
3294 			t = UploadFileThread(args)
3295 			upload_threads.append(t)
3296 			t.start()
3297 
3298 		def WaitForUploads():			
3299 			for t in upload_threads:
3300 				t.join()
3301 
3302 		patches = dict()
3303 		[patches.setdefault(v, k) for k, v in patch_list]
3304 		for filename in patches.keys():
3305 			base_content, new_content, is_binary, status = files[filename]
3306 			file_id_str = patches.get(filename)
3307 			if file_id_str.find("nobase") != -1:
3308 				base_content = None
3309 				file_id_str = file_id_str[file_id_str.rfind("_") + 1:]
3310 			file_id = int(file_id_str)
3311 			if base_content != None:
3312 				StartUploadFile(filename, file_id, base_content, is_binary, status, True)
3313 			if new_content != None:
3314 				StartUploadFile(filename, file_id, new_content, is_binary, status, False)
3315 		WaitForUploads()
3316 
3317 	def IsImage(self, filename):
3318 		"""Returns true if the filename has an image extension."""
3319 		mimetype =  mimetypes.guess_type(filename)[0]
3320 		if not mimetype:
3321 			return False
3322 		return mimetype.startswith("image/")
3323 
3324 	def IsBinary(self, filename):
3325 		"""Returns true if the guessed mimetyped isnt't in text group."""
3326 		mimetype = mimetypes.guess_type(filename)[0]
3327 		if not mimetype:
3328 			return False  # e.g. README, "real" binaries usually have an extension
3329 		# special case for text files which don't start with text/
3330 		if mimetype in TEXT_MIMETYPES:
3331 			return False
3332 		return not mimetype.startswith("text/")
3333 
3334 
3335 class FakeMercurialUI(object):
3336 	def __init__(self):
3337 		self.quiet = True
3338 		self.output = ''
3339 	
3340 	def write(self, *args, **opts):
3341 		self.output += ' '.join(args)
3342 	def copy(self):
3343 		return self
3344 	def status(self, *args, **opts):
3345 		pass
3346 
3347 	def formatter(self, topic, opts):
3348 		from mercurial.formatter import plainformatter
3349 		return plainformatter(self, topic, opts)
3350 	
3351 	def readconfig(self, *args, **opts):
3352 		pass
3353 	def expandpath(self, *args, **opts):
3354 		return global_ui.expandpath(*args, **opts)
3355 	def configitems(self, *args, **opts):
3356 		return global_ui.configitems(*args, **opts)
3357 	def config(self, *args, **opts):
3358 		return global_ui.config(*args, **opts)
3359 
3360 use_hg_shell = False	# set to True to shell out to hg always; slower
3361 
3362 class MercurialVCS(VersionControlSystem):
3363 	"""Implementation of the VersionControlSystem interface for Mercurial."""
3364 
3365 	def __init__(self, options, ui, repo):
3366 		super(MercurialVCS, self).__init__(options)
3367 		self.ui = ui
3368 		self.repo = repo
3369 		self.status = None
3370 		# Absolute path to repository (we can be in a subdir)
3371 		self.repo_dir = os.path.normpath(repo.root)
3372 		# Compute the subdir
3373 		cwd = os.path.normpath(os.getcwd())
3374 		assert cwd.startswith(self.repo_dir)
3375 		self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/")
3376 		if self.options.revision:
3377 			self.base_rev = self.options.revision
3378 		else:
3379 			mqparent, err = RunShellWithReturnCode(['hg', 'log', '--rev', 'qparent', '--template={node}'])
3380 			if not err and mqparent != "":
3381 				self.base_rev = mqparent
3382 			else:
3383 				out = RunShell(["hg", "parents", "-q"], silent_ok=True).strip()
3384 				if not out:
3385 					# No revisions; use 0 to mean a repository with nothing.
3386 					out = "0:0"
3387 				self.base_rev = out.split(':')[1].strip()
3388 	def _GetRelPath(self, filename):
3389 		"""Get relative path of a file according to the current directory,
3390 		given its logical path in the repo."""
3391 		assert filename.startswith(self.subdir), (filename, self.subdir)
3392 		return filename[len(self.subdir):].lstrip(r"\/")
3393 
3394 	def GenerateDiff(self, extra_args):
3395 		# If no file specified, restrict to the current subdir
3396 		extra_args = extra_args or ["."]
3397 		cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
3398 		data = RunShell(cmd, silent_ok=True)
3399 		svndiff = []
3400 		filecount = 0
3401 		for line in data.splitlines():
3402 			m = re.match("diff --git a/(\S+) b/(\S+)", line)
3403 			if m:
3404 				# Modify line to make it look like as it comes from svn diff.
3405 				# With this modification no changes on the server side are required
3406 				# to make upload.py work with Mercurial repos.
3407 				# NOTE: for proper handling of moved/copied files, we have to use
3408 				# the second filename.
3409 				filename = m.group(2)
3410 				svndiff.append("Index: %s" % filename)
3411 				svndiff.append("=" * 67)
3412 				filecount += 1
3413 				logging.info(line)
3414 			else:
3415 				svndiff.append(line)
3416 		if not filecount:
3417 			ErrorExit("No valid patches found in output from hg diff")
3418 		return "\n".join(svndiff) + "\n"
3419 
3420 	def GetUnknownFiles(self):
3421 		"""Return a list of files unknown to the VCS."""
3422 		args = []
3423 		status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
3424 				silent_ok=True)
3425 		unknown_files = []
3426 		for line in status.splitlines():
3427 			st, fn = line.split(" ", 1)
3428 			if st == "?":
3429 				unknown_files.append(fn)
3430 		return unknown_files
3431 
3432 	def get_hg_status(self, rev, path):
3433 		# We'd like to use 'hg status -C path', but that is buggy
3434 		# (see http://mercurial.selenic.com/bts/issue3023).
3435 		# Instead, run 'hg status -C' without a path
3436 		# and skim the output for the path we want.
3437 		if self.status is None:
3438 			if use_hg_shell:
3439 				out = RunShell(["hg", "status", "-C", "--rev", rev])
3440 			else:
3441 				fui = FakeMercurialUI()
3442 				ret = hg_commands.status(fui, self.repo, *[], **{'rev': [rev], 'copies': True})
3443 				if ret:
3444 					raise hg_util.Abort(ret)
3445 				out = fui.output
3446 			self.status = out.splitlines()
3447 		for i in range(len(self.status)):
3448 			# line is
3449 			#	A path
3450 			#	M path
3451 			# etc
3452 			line = to_slash(self.status[i])
3453 			if line[2:] == path:
3454 				if i+1 < len(self.status) and self.status[i+1][:2] == '  ':
3455 					return self.status[i:i+2]
3456 				return self.status[i:i+1]
3457 		raise hg_util.Abort("no status for " + path)
3458 	
3459 	def GetBaseFile(self, filename):
3460 		set_status("inspecting " + filename)
3461 		# "hg status" and "hg cat" both take a path relative to the current subdir
3462 		# rather than to the repo root, but "hg diff" has given us the full path
3463 		# to the repo root.
3464 		base_content = ""
3465 		new_content = None
3466 		is_binary = False
3467 		oldrelpath = relpath = self._GetRelPath(filename)
3468 		out = self.get_hg_status(self.base_rev, relpath)
3469 		status, what = out[0].split(' ', 1)
3470 		if len(out) > 1 and status == "A" and what == relpath:
3471 			oldrelpath = out[1].strip()
3472 			status = "M"
3473 		if ":" in self.base_rev:
3474 			base_rev = self.base_rev.split(":", 1)[0]
3475 		else:
3476 			base_rev = self.base_rev
3477 		if status != "A":
3478 			if use_hg_shell:
3479 				base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath], silent_ok=True)
3480 			else:
3481 				base_content = str(self.repo[base_rev][oldrelpath].data())
3482 			is_binary = "\0" in base_content  # Mercurial's heuristic
3483 		if status != "R":
3484 			new_content = open(relpath, "rb").read()
3485 			is_binary = is_binary or "\0" in new_content
3486 		if is_binary and base_content and use_hg_shell:
3487 			# Fetch again without converting newlines
3488 			base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
3489 				silent_ok=True, universal_newlines=False)
3490 		if not is_binary or not self.IsImage(relpath):
3491 			new_content = None
3492 		return base_content, new_content, is_binary, status
3493 
3494 
3495 # NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
3496 def SplitPatch(data):
3497 	"""Splits a patch into separate pieces for each file.
3498 
3499 	Args:
3500 		data: A string containing the output of svn diff.
3501 
3502 	Returns:
3503 		A list of 2-tuple (filename, text) where text is the svn diff output
3504 			pertaining to filename.
3505 	"""
3506 	patches = []
3507 	filename = None
3508 	diff = []
3509 	for line in data.splitlines(True):
3510 		new_filename = None
3511 		if line.startswith('Index:'):
3512 			unused, new_filename = line.split(':', 1)
3513 			new_filename = new_filename.strip()
3514 		elif line.startswith('Property changes on:'):
3515 			unused, temp_filename = line.split(':', 1)
3516 			# When a file is modified, paths use '/' between directories, however
3517 			# when a property is modified '\' is used on Windows.  Make them the same
3518 			# otherwise the file shows up twice.
3519 			temp_filename = to_slash(temp_filename.strip())
3520 			if temp_filename != filename:
3521 				# File has property changes but no modifications, create a new diff.
3522 				new_filename = temp_filename
3523 		if new_filename:
3524 			if filename and diff:
3525 				patches.append((filename, ''.join(diff)))
3526 			filename = new_filename
3527 			diff = [line]
3528 			continue
3529 		if diff is not None:
3530 			diff.append(line)
3531 	if filename and diff:
3532 		patches.append((filename, ''.join(diff)))
3533 	return patches
3534 
3535 
3536 def UploadSeparatePatches(issue, rpc_server, patchset, data, options):
3537 	"""Uploads a separate patch for each file in the diff output.
3538 
3539 	Returns a list of [patch_key, filename] for each file.
3540 	"""
3541 	patches = SplitPatch(data)
3542 	rv = []
3543 	for patch in patches:
3544 		set_status("uploading patch for " + patch[0])
3545 		if len(patch[1]) > MAX_UPLOAD_SIZE:
3546 			print ("Not uploading the patch for " + patch[0] +
3547 				" because the file is too large.")
3548 			continue
3549 		form_fields = [("filename", patch[0])]
3550 		if not options.download_base:
3551 			form_fields.append(("content_upload", "1"))
3552 		files = [("data", "data.diff", patch[1])]
3553 		ctype, body = EncodeMultipartFormData(form_fields, files)
3554 		url = "/%d/upload_patch/%d" % (int(issue), int(patchset))
3555 		print "Uploading patch for " + patch[0]
3556 		response_body = rpc_server.Send(url, body, content_type=ctype)
3557 		lines = response_body.splitlines()
3558 		if not lines or lines[0] != "OK":
3559 			StatusUpdate("  --> %s" % response_body)
3560 			sys.exit(1)
3561 		rv.append([lines[1], patch[0]])
3562 	return rv