1 #!/usr/bin/env python
2 #
3 # Copyright 2007-2009 Google Inc.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 #     http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 
17 '''Mercurial interface to codereview.appspot.com.
18 
19 To configure, set the following options in
20 your repository's .hg/hgrc file.
21 
22 	[extensions]
23 	codereview = path/to/codereview.py
24 
25 	[codereview]
26 	server = codereview.appspot.com
27 
28 The server should be running Rietveld; see http://code.google.com/p/rietveld/.
29 
30 In addition to the new commands, this extension introduces
31 the file pattern syntax @nnnnnn, where nnnnnn is a change list
32 number, to mean the files included in that change list, which
33 must be associated with the current client.
34 
35 For example, if change 123456 contains the files x.go and y.go,
36 "hg diff @123456" is equivalent to"hg diff x.go y.go".
37 '''
38 
39 from mercurial import cmdutil, commands, hg, util, error, match
40 from mercurial.node import nullrev, hex, nullid, short
41 import os, re, time
42 import stat
43 import subprocess
44 import threading
45 from HTMLParser import HTMLParser
46 try:
47 	from xml.etree import ElementTree as ET
48 except:
49 	from elementtree import ElementTree as ET
50 
51 try:
52 	hgversion = util.version()
53 except:
54 	from mercurial.version import version as v
55 	hgversion = v.get_version()
56 
57 try:
58 	from mercurial.discovery import findcommonincoming
59 except:
60 	def findcommonincoming(repo, remote):
61 		return repo.findcommonincoming(remote)
62 
63 oldMessage = """
64 The code review extension requires Mercurial 1.3 or newer.
65 
66 To install a new Mercurial,
67 
68 	sudo easy_install mercurial
69 
70 works on most systems.
71 """
72 
73 linuxMessage = """
74 You may need to clear your current Mercurial installation by running:
75 
76 	sudo apt-get remove mercurial mercurial-common
77 	sudo rm -rf /etc/mercurial
78 """
79 
80 if hgversion < '1.3':
81 	msg = oldMessage
82 	if os.access("/etc/mercurial", 0):
83 		msg += linuxMessage
84 	raise util.Abort(msg)
85 
86 def promptyesno(ui, msg):
87 	# Arguments to ui.prompt changed between 1.3 and 1.3.1.
88 	# Even so, some 1.3.1 distributions seem to have the old prompt!?!?
89 	# What a terrible way to maintain software.
90 	try:
91 		return ui.promptchoice(msg, ["&yes", "&no"], 0) == 0
92 	except AttributeError:
93 		return ui.prompt(msg, ["&yes", "&no"], "y") != "n"
94 
95 # To experiment with Mercurial in the python interpreter:
96 #    >>> repo = hg.repository(ui.ui(), path = ".")
97 
98 #######################################################################
99 # Normally I would split this into multiple files, but it simplifies
100 # import path headaches to keep it all in one file.  Sorry.
101 
102 import sys
103 if __name__ == "__main__":
104 	print >>sys.stderr, "This is a Mercurial extension and should not be invoked directly."
105 	sys.exit(2)
106 
107 server = "codereview.appspot.com"
108 server_url_base = None
109 defaultcc = None
110 contributors = {}
111 
112 #######################################################################
113 # Change list parsing.
114 #
115 # Change lists are stored in .hg/codereview/cl.nnnnnn
116 # where nnnnnn is the number assigned by the code review server.
117 # Most data about a change list is stored on the code review server
118 # too: the description, reviewer, and cc list are all stored there.
119 # The only thing in the cl.nnnnnn file is the list of relevant files.
120 # Also, the existence of the cl.nnnnnn file marks this repository
121 # as the one where the change list lives.
122 
123 emptydiff = """Index: ~rietveld~placeholder~
124 ===================================================================
125 diff --git a/~rietveld~placeholder~ b/~rietveld~placeholder~
126 new file mode 100644
127 """
128 
129 
130 class CL(object):
131 	def __init__(self, name):
132 		self.name = name
133 		self.desc = ''
134 		self.files = []
135 		self.reviewer = []
136 		self.cc = []
137 		self.url = ''
138 		self.local = False
139 		self.web = False
140 		self.copied_from = None	# None means current user
141 		self.mailed = False
142 
143 	def DiskText(self):
144 		cl = self
145 		s = ""
146 		if cl.copied_from:
147 			s += "Author: " + cl.copied_from + "\n\n"
148 		s += "Mailed: " + str(self.mailed) + "\n"
149 		s += "Description:\n"
150 		s += Indent(cl.desc, "\t")
151 		s += "Files:\n"
152 		for f in cl.files:
153 			s += "\t" + f + "\n"
154 		return s
155 
156 	def EditorText(self):
157 		cl = self
158 		s = _change_prolog
159 		s += "\n"
160 		if cl.copied_from:
161 			s += "Author: " + cl.copied_from + "\n"
162 		if cl.url != '':
163 			s += 'URL: ' + cl.url + '	# cannot edit\n\n'
164 		s += "Reviewer: " + JoinComma(cl.reviewer) + "\n"
165 		s += "CC: " + JoinComma(cl.cc) + "\n"
166 		s += "\n"
167 		s += "Description:\n"
168 		if cl.desc == '':
169 			s += "\t<enter description here>\n"
170 		else:
171 			s += Indent(cl.desc, "\t")
172 		s += "\n"
173 		if cl.local or cl.name == "new":
174 			s += "Files:\n"
175 			for f in cl.files:
176 				s += "\t" + f + "\n"
177 			s += "\n"
178 		return s
179 
180 	def PendingText(self):
181 		cl = self
182 		s = cl.name + ":" + "\n"
183 		s += Indent(cl.desc, "\t")
184 		s += "\n"
185 		if cl.copied_from:
186 			s += "\tAuthor: " + cl.copied_from + "\n"
187 		s += "\tReviewer: " + JoinComma(cl.reviewer) + "\n"
188 		s += "\tCC: " + JoinComma(cl.cc) + "\n"
189 		s += "\tFiles:\n"
190 		for f in cl.files:
191 			s += "\t\t" + f + "\n"
192 		return s
193 
194 	def Flush(self, ui, repo):
195 		if self.name == "new":
196 			self.Upload(ui, repo, gofmt_just_warn=True)
197 		dir = CodeReviewDir(ui, repo)
198 		path = dir + '/cl.' + self.name
199 		f = open(path+'!', "w")
200 		f.write(self.DiskText())
201 		f.close()
202 		if sys.platform == "win32" and os.path.isfile(path):
203 			os.remove(path)
204 		os.rename(path+'!', path)
205 		if self.web and not self.copied_from:
206 			EditDesc(self.name, desc=self.desc,
207 				reviewers=JoinComma(self.reviewer), cc=JoinComma(self.cc))
208 
209 	def Delete(self, ui, repo):
210 		dir = CodeReviewDir(ui, repo)
211 		os.unlink(dir + "/cl." + self.name)
212 
213 	def Subject(self):
214 		s = line1(self.desc)
215 		if len(s) > 60:
216 			s = s[0:55] + "..."
217 		if self.name != "new":
218 			s = "code review %s: %s" % (self.name, s)
219 		return s
220 
221 	def Upload(self, ui, repo, send_mail=False, gofmt=True, gofmt_just_warn=False):
222 		if not self.files:
223 			ui.warn("no files in change list\n")
224 		if ui.configbool("codereview", "force_gofmt", True) and gofmt:
225 			CheckGofmt(ui, repo, self.files, just_warn=gofmt_just_warn)
226 		os.chdir(repo.root)
227 		form_fields = [
228 			("content_upload", "1"),
229 			("reviewers", JoinComma(self.reviewer)),
230 			("cc", JoinComma(self.cc)),
231 			("description", self.desc),
232 			("base_hashes", ""),
233 			# Would prefer not to change the subject
234 			# on reupload, but /upload requires it.
235 			("subject", self.Subject()),
236 		]
237 
238 		# NOTE(rsc): This duplicates too much of RealMain,
239 		# but RealMain doesn't have the most reusable interface.
240 		if self.name != "new":
241 			form_fields.append(("issue", self.name))
242 		vcs = None
243 		if self.files:
244 			vcs = GuessVCS(upload_options)
245 			data = vcs.GenerateDiff(self.files)
246 			files = vcs.GetBaseFiles(data)
247 			if len(data) > MAX_UPLOAD_SIZE:
248 				uploaded_diff_file = []
249 				form_fields.append(("separate_patches", "1"))
250 			else:
251 				uploaded_diff_file = [("data", "data.diff", data)]
252 		else:
253 			uploaded_diff_file = [("data", "data.diff", emptydiff)]
254 		ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
255 		response_body = MySend("/upload", body, content_type=ctype)
256 		patchset = None
257 		msg = response_body
258 		lines = msg.splitlines()
259 		if len(lines) >= 2:
260 			msg = lines[0]
261 			patchset = lines[1].strip()
262 			patches = [x.split(" ", 1) for x in lines[2:]]
263 		ui.status(msg + "\n")
264 		if not response_body.startswith("Issue created.") and not response_body.startswith("Issue updated."):
265 			raise util.Abort("failed to update issue: " + response_body)
266 		issue = msg[msg.rfind("/")+1:]
267 		self.name = issue
268 		if not self.url:
269 			self.url = server_url_base + self.name
270 		if not uploaded_diff_file:
271 			patches = UploadSeparatePatches(issue, rpc, patchset, data, upload_options)
272 		if vcs:
273 			vcs.UploadBaseFiles(issue, rpc, patches, patchset, upload_options, files)
274 		if send_mail:
275 			MySend("/" + issue + "/mail", payload="")
276 		self.web = True
277 		self.Flush(ui, repo)
278 		return
279 
280 	def Mail(self, ui,repo):
281 		pmsg = "Hello " + JoinComma(self.reviewer)
282 		if self.cc:
283 			pmsg += " (cc: %s)" % (', '.join(self.cc),)
284 		pmsg += ",\n"
285 		pmsg += "\n"
286 		if not self.mailed:
287 			pmsg += "I'd like you to review this change.\n"
288 		else:
289 			pmsg += "Please take another look.\n"
290 		PostMessage(ui, self.name, pmsg, subject=self.Subject())
291 		self.mailed = True
292 		self.Flush(ui, repo)
293 
294 def GoodCLName(name):
295 	return re.match("^[0-9]+$", name)
296 
297 def ParseCL(text, name):
298 	sname = None
299 	lineno = 0
300 	sections = {
301 		'Author': '',
302 		'Description': '',
303 		'Files': '',
304 		'URL': '',
305 		'Reviewer': '',
306 		'CC': '',
307 		'Mailed': '',
308 	}
309 	for line in text.split('\n'):
310 		lineno += 1
311 		line = line.rstrip()
312 		if line != '' and line[0] == '#':
313 			continue
314 		if line == '' or line[0] == ' ' or line[0] == '\t':
315 			if sname == None and line != '':
316 				return None, lineno, 'text outside section'
317 			if sname != None:
318 				sections[sname] += line + '\n'
319 			continue
320 		p = line.find(':')
321 		if p >= 0:
322 			s, val = line[:p].strip(), line[p+1:].strip()
323 			if s in sections:
324 				sname = s
325 				if val != '':
326 					sections[sname] += val + '\n'
327 				continue
328 		return None, lineno, 'malformed section header'
329 
330 	for k in sections:
331 		sections[k] = StripCommon(sections[k]).rstrip()
332 
333 	cl = CL(name)
334 	if sections['Author']:
335 		cl.copied_from = sections['Author']
336 	cl.desc = sections['Description']
337 	for line in sections['Files'].split('\n'):
338 		i = line.find('#')
339 		if i >= 0:
340 			line = line[0:i].rstrip()
341 		if line == '':
342 			continue
343 		cl.files.append(line)
344 	cl.reviewer = SplitCommaSpace(sections['Reviewer'])
345 	cl.cc = SplitCommaSpace(sections['CC'])
346 	cl.url = sections['URL']
347 	if sections['Mailed'] != 'False':
348 		# Odd default, but avoids spurious mailings when
349 		# reading old CLs that do not have a Mailed: line.
350 		# CLs created with this update will always have
351 		# Mailed: False on disk.
352 		cl.mailed = True
353 	if cl.desc == '<enter description here>':
354 		cl.desc = ''
355 	return cl, 0, ''
356 
357 def SplitCommaSpace(s):
358 	s = s.strip()
359 	if s == "":
360 		return []
361 	return re.split(", *", s)
362 
363 def CutDomain(s):
364 	i = s.find('@')
365 	if i >= 0:
366 		s = s[0:i]
367 	return s
368 
369 def JoinComma(l):
370 	return ", ".join(l)
371 
372 def ExceptionDetail():
373 	s = str(sys.exc_info()[0])
374 	if s.startswith("<type '") and s.endswith("'>"):
375 		s = s[7:-2]
376 	elif s.startswith("<class '") and s.endswith("'>"):
377 		s = s[8:-2]
378 	arg = str(sys.exc_info()[1])
379 	if len(arg) > 0:
380 		s += ": " + arg
381 	return s
382 
383 def IsLocalCL(ui, repo, name):
384 	return GoodCLName(name) and os.access(CodeReviewDir(ui, repo) + "/cl." + name, 0)
385 
386 # Load CL from disk and/or the web.
387 def LoadCL(ui, repo, name, web=True):
388 	if not GoodCLName(name):
389 		return None, "invalid CL name"
390 	dir = CodeReviewDir(ui, repo)
391 	path = dir + "cl." + name
392 	if os.access(path, 0):
393 		ff = open(path)
394 		text = ff.read()
395 		ff.close()
396 		cl, lineno, err = ParseCL(text, name)
397 		if err != "":
398 			return None, "malformed CL data: "+err
399 		cl.local = True
400 	else:
401 		cl = CL(name)
402 	if web:
403 		try:
404 			f = GetSettings(name)
405 		except:
406 			return None, "cannot load CL %s from code review server: %s" % (name, ExceptionDetail())
407 		if 'reviewers' not in f:
408 			return None, "malformed response loading CL data from code review server"
409 		cl.reviewer = SplitCommaSpace(f['reviewers'])
410 		cl.cc = SplitCommaSpace(f['cc'])
411 		if cl.local and cl.copied_from and cl.desc:
412 			# local copy of CL written by someone else
413 			# and we saved a description.  use that one,
414 			# so that committers can edit the description
415 			# before doing hg submit.
416 			pass
417 		else:
418 			cl.desc = f['description']
419 		cl.url = server_url_base + name
420 		cl.web = True
421 	return cl, ''
422 
423 class LoadCLThread(threading.Thread):
424 	def __init__(self, ui, repo, dir, f, web):
425 		threading.Thread.__init__(self)
426 		self.ui = ui
427 		self.repo = repo
428 		self.dir = dir
429 		self.f = f
430 		self.web = web
431 		self.cl = None
432 	def run(self):
433 		cl, err = LoadCL(self.ui, self.repo, self.f[3:], web=self.web)
434 		if err != '':
435 			self.ui.warn("loading "+self.dir+self.f+": " + err + "\n")
436 			return
437 		self.cl = cl
438 
439 # Load all the CLs from this repository.
440 def LoadAllCL(ui, repo, web=True):
441 	dir = CodeReviewDir(ui, repo)
442 	m = {}
443 	files = [f for f in os.listdir(dir) if f.startswith('cl.')]
444 	if not files:
445 		return m
446 	active = []
447 	first = True
448 	for f in files:
449 		t = LoadCLThread(ui, repo, dir, f, web)
450 		t.start()
451 		if web and first:
452 			# first request: wait in case it needs to authenticate
453 			# otherwise we get lots of user/password prompts
454 			# running in parallel.
455 			t.join()
456 			if t.cl:
457 				m[t.cl.name] = t.cl
458 			first = False
459 		else:
460 			active.append(t)
461 	for t in active:
462 		t.join()
463 		if t.cl:
464 			m[t.cl.name] = t.cl
465 	return m
466 
467 # Find repository root.  On error, ui.warn and return None
468 def RepoDir(ui, repo):
469 	url = repo.url();
470 	if not url.startswith('file:'):
471 		ui.warn("repository %s is not in local file system\n" % (url,))
472 		return None
473 	url = url[5:]
474 	if url.endswith('/'):
475 		url = url[:-1]
476 	return url
477 
478 # Find (or make) code review directory.  On error, ui.warn and return None
479 def CodeReviewDir(ui, repo):
480 	dir = RepoDir(ui, repo)
481 	if dir == None:
482 		return None
483 	dir += '/.hg/codereview/'
484 	if not os.path.isdir(dir):
485 		try:
486 			os.mkdir(dir, 0700)
487 		except:
488 			ui.warn('cannot mkdir %s: %s\n' % (dir, ExceptionDetail()))
489 			return None
490 	return dir
491 
492 # Strip maximal common leading white space prefix from text
493 def StripCommon(text):
494 	ws = None
495 	for line in text.split('\n'):
496 		line = line.rstrip()
497 		if line == '':
498 			continue
499 		white = line[:len(line)-len(line.lstrip())]
500 		if ws == None:
501 			ws = white
502 		else:
503 			common = ''
504 			for i in range(min(len(white), len(ws))+1):
505 				if white[0:i] == ws[0:i]:
506 					common = white[0:i]
507 			ws = common
508 		if ws == '':
509 			break
510 	if ws == None:
511 		return text
512 	t = ''
513 	for line in text.split('\n'):
514 		line = line.rstrip()
515 		if line.startswith(ws):
516 			line = line[len(ws):]
517 		if line == '' and t == '':
518 			continue
519 		t += line + '\n'
520 	while len(t) >= 2 and t[-2:] == '\n\n':
521 		t = t[:-1]
522 	return t
523 
524 # Indent text with indent.
525 def Indent(text, indent):
526 	t = ''
527 	for line in text.split('\n'):
528 		t += indent + line + '\n'
529 	return t
530 
531 # Return the first line of l
532 def line1(text):
533 	return text.split('\n')[0]
534 
535 _change_prolog = """# Change list.
536 # Lines beginning with # are ignored.
537 # Multi-line values should be indented.
538 """
539 
540 #######################################################################
541 # Mercurial helper functions
542 
543 # Get effective change nodes taking into account applied MQ patches
544 def effective_revpair(repo):
545     try:
546 	return cmdutil.revpair(repo, ['qparent'])
547     except:
548 	return cmdutil.revpair(repo, None)
549 
550 # Return list of changed files in repository that match pats.
551 def ChangedFiles(ui, repo, pats, opts):
552 	# Find list of files being operated on.
553 	matcher = cmdutil.match(repo, pats, opts)
554 	node1, node2 = effective_revpair(repo)
555 	modified, added, removed = repo.status(node1, node2, matcher)[:3]
556 	l = modified + added + removed
557 	l.sort()
558 	return l
559 
560 # Return list of changed files in repository that match pats and still exist.
561 def ChangedExistingFiles(ui, repo, pats, opts):
562 	matcher = cmdutil.match(repo, pats, opts)
563 	node1, node2 = effective_revpair(repo)
564 	modified, added, _ = repo.status(node1, node2, matcher)[:3]
565 	l = modified + added
566 	l.sort()
567 	return l
568 
569 # Return list of files claimed by existing CLs
570 def TakenFiles(ui, repo):
571 	return Taken(ui, repo).keys()
572 
573 def Taken(ui, repo):
574 	all = LoadAllCL(ui, repo, web=False)
575 	taken = {}
576 	for _, cl in all.items():
577 		for f in cl.files:
578 			taken[f] = cl
579 	return taken
580 
581 # Return list of changed files that are not claimed by other CLs
582 def DefaultFiles(ui, repo, pats, opts):
583 	return Sub(ChangedFiles(ui, repo, pats, opts), TakenFiles(ui, repo))
584 
585 def Sub(l1, l2):
586 	return [l for l in l1 if l not in l2]
587 
588 def Add(l1, l2):
589 	l = l1 + Sub(l2, l1)
590 	l.sort()
591 	return l
592 
593 def Intersect(l1, l2):
594 	return [l for l in l1 if l in l2]
595 
596 def getremote(ui, repo, opts):
597 	# save $http_proxy; creating the HTTP repo object will
598 	# delete it in an attempt to "help"
599 	proxy = os.environ.get('http_proxy')
600 	source = hg.parseurl(ui.expandpath("default"), None)[0]
601 	try:
602 		remoteui = hg.remoteui # hg 1.6
603         except:
604 		remoteui = cmdutil.remoteui
605 	other = hg.repository(remoteui(repo, opts), source)
606 	if proxy is not None:
607 		os.environ['http_proxy'] = proxy
608 	return other
609 
610 def Incoming(ui, repo, opts):
611 	_, incoming, _ = findcommonincoming(repo, getremote(ui, repo, opts))
612 	return incoming
613 
614 def EditCL(ui, repo, cl):
615 	s = cl.EditorText()
616 	while True:
617 		s = ui.edit(s, ui.username())
618 		clx, line, err = ParseCL(s, cl.name)
619 		if err != '':
620 			if not promptyesno(ui, "error parsing change list: line %d: %s\nre-edit (y/n)?" % (line, err)):
621 				return "change list not modified"
622 			continue
623 		cl.desc = clx.desc;
624 		cl.reviewer = clx.reviewer
625 		cl.cc = clx.cc
626 		cl.files = clx.files
627 		if cl.desc == '':
628 			if promptyesno(ui, "change list should have description\nre-edit (y/n)?"):
629 				continue
630 		break
631 	return ""
632 
633 # For use by submit, etc. (NOT by change)
634 # Get change list number or list of files from command line.
635 # If files are given, make a new change list.
636 def CommandLineCL(ui, repo, pats, opts, defaultcc=None):
637 	if len(pats) > 0 and GoodCLName(pats[0]):
638 		if len(pats) != 1:
639 			return None, "cannot specify change number and file names"
640 		if opts.get('message'):
641 			return None, "cannot use -m with existing CL"
642 		cl, err = LoadCL(ui, repo, pats[0], web=True)
643 		if err != "":
644 			return None, err
645 	else:
646 		cl = CL("new")
647 		cl.local = True
648 		cl.files = Sub(ChangedFiles(ui, repo, pats, opts), TakenFiles(ui, repo))
649 		if not cl.files:
650 			return None, "no files changed"
651 	if opts.get('reviewer'):
652 		cl.reviewer = Add(cl.reviewer, SplitCommaSpace(opts.get('reviewer')))
653 	if opts.get('cc'):
654 		cl.cc = Add(cl.cc, SplitCommaSpace(opts.get('cc')))
655 	if defaultcc:
656 		cl.cc = Add(cl.cc, defaultcc)
657 	if cl.name == "new":
658 		if opts.get('message'):
659 			cl.desc = opts.get('message')
660 		else:
661 			err = EditCL(ui, repo, cl)
662 			if err != '':
663 				return None, err
664 	return cl, ""
665 
666 # reposetup replaces cmdutil.match with this wrapper,
667 # which expands the syntax @clnumber to mean the files
668 # in that CL.
669 original_match = None
670 def ReplacementForCmdutilMatch(repo, pats=[], opts={}, globbed=False, default='relpath'):
671 	taken = []
672 	files = []
673         pats = pats or []
674 	for p in pats:
675 		if p.startswith('@'):
676 			taken.append(p)
677 			clname = p[1:]
678 			if not GoodCLName(clname):
679 				raise util.Abort("invalid CL name " + clname)
680 			cl, err = LoadCL(repo.ui, repo, clname, web=False)
681 			if err != '':
682 				raise util.Abort("loading CL " + clname + ": " + err)
683 			if cl.files == None:
684 				raise util.Abort("no files in CL " + clname)
685 			files = Add(files, cl.files)
686 	pats = Sub(pats, taken) + ['path:'+f for f in files]
687 	return original_match(repo, pats=pats, opts=opts, globbed=globbed, default=default)
688 
689 def RelativePath(path, cwd):
690 	n = len(cwd)
691 	if path.startswith(cwd) and path[n] == '/':
692 		return path[n+1:]
693 	return path
694 
695 # Check that gofmt run on the list of files does not change them
696 def CheckGofmt(ui, repo, files, just_warn=False):
697 	files = [f for f in files if (f.startswith('src/') or f.startswith('test/bench/')) and f.endswith('.go')]
698 	if not files:
699 		return
700 	cwd = os.getcwd()
701 	files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
702 	files = [f for f in files if os.access(f, 0)]
703 	if not files:
704 		return
705 	try:
706 		cmd = subprocess.Popen(["gofmt", "-l"] + files, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
707 		cmd.stdin.close()
708 	except:
709 		raise util.Abort("gofmt: " + ExceptionDetail())
710 	data = cmd.stdout.read()
711 	errors = cmd.stderr.read()
712 	cmd.wait()
713 	if len(errors) > 0:
714 		ui.warn("gofmt errors:\n" + errors.rstrip() + "\n")
715 		return
716 	if len(data) > 0:
717 		msg = "gofmt needs to format these files (run hg gofmt):\n" + Indent(data, "\t").rstrip()
718 		if just_warn:
719 			ui.warn("warning: " + msg + "\n")
720 		else:
721 			raise util.Abort(msg)
722 	return
723 
724 #######################################################################
725 # Mercurial commands
726 
727 # every command must take a ui and and repo as arguments.
728 # opts is a dict where you can find other command line flags
729 #
730 # Other parameters are taken in order from items on the command line that
731 # don't start with a dash.  If no default value is given in the parameter list,
732 # they are required.
733 #
734 
735 def change(ui, repo, *pats, **opts):
736 	"""create, edit or delete a change list
737 
738 	Create, edit or delete a change list.
739 	A change list is a group of files to be reviewed and submitted together,
740 	plus a textual description of the change.
741 	Change lists are referred to by simple alphanumeric names.
742 
743 	Changes must be reviewed before they can be submitted.
744 
745 	In the absence of options, the change command opens the
746 	change list for editing in the default editor.
747 
748 	Deleting a change with the -d or -D flag does not affect
749 	the contents of the files listed in that change.  To revert
750 	the files listed in a change, use
751 
752 		hg revert @123456
753 
754 	before running hg change -d 123456.
755 	"""
756 
757 	dirty = {}
758 	if len(pats) > 0 and GoodCLName(pats[0]):
759 		name = pats[0]
760 		if len(pats) != 1:
761 			return "cannot specify CL name and file patterns"
762 		pats = pats[1:]
763 		cl, err = LoadCL(ui, repo, name, web=True)
764 		if err != '':
765 			return err
766 		if not cl.local and (opts["stdin"] or not opts["stdout"]):
767 			return "cannot change non-local CL " + name
768 	else:
769 		name = "new"
770 		cl = CL("new")
771 		dirty[cl] = True
772 		files = ChangedFiles(ui, repo, pats, opts)
773 		taken = TakenFiles(ui, repo)
774 		files = Sub(files, taken)
775 
776 	if opts["delete"] or opts["deletelocal"]:
777 		if opts["delete"] and opts["deletelocal"]:
778 			return "cannot use -d and -D together"
779 		flag = "-d"
780 		if opts["deletelocal"]:
781 			flag = "-D"
782 		if name == "new":
783 			return "cannot use "+flag+" with file patterns"
784 		if opts["stdin"] or opts["stdout"]:
785 			return "cannot use "+flag+" with -i or -o"
786 		if not cl.local:
787 			return "cannot change non-local CL " + name
788 		if opts["delete"]:
789 			if cl.copied_from:
790 				return "original author must delete CL; hg change -D will remove locally"
791 			PostMessage(ui, cl.name, "*** Abandoned ***")
792 			EditDesc(cl.name, closed="checked")
793 		cl.Delete(ui, repo)
794 		return
795 
796 	if opts["stdin"]:
797 		s = sys.stdin.read()
798 		clx, line, err = ParseCL(s, name)
799 		if err != '':
800 			return "error parsing change list: line %d: %s" % (line, err)
801 		if clx.desc is not None:
802 			cl.desc = clx.desc;
803 			dirty[cl] = True
804 		if clx.reviewer is not None:
805 			cl.reviewer = clx.reviewer
806 			dirty[cl] = True
807 		if clx.cc is not None:
808 			cl.cc = clx.cc
809 			dirty[cl] = True
810 		if clx.files is not None:
811 			cl.files = clx.files
812 			dirty[cl] = True
813 
814 	if not opts["stdin"] and not opts["stdout"]:
815 		if name == "new":
816 			cl.files = files
817 		err = EditCL(ui, repo, cl)
818 		if err != "":
819 			return err
820 		dirty[cl] = True
821 
822 	for d, _ in dirty.items():
823 		d.Flush(ui, repo)
824 
825 	if opts["stdout"]:
826 		ui.write(cl.EditorText())
827 	elif name == "new":
828 		if ui.quiet:
829 			ui.write(cl.name)
830 		else:
831 			ui.write("CL created: " + cl.url + "\n")
832 	return
833 
834 def code_login(ui, repo, **opts):
835 	"""log in to code review server
836 
837 	Logs in to the code review server, saving a cookie in
838 	a file in your home directory.
839 	"""
840 	MySend(None)
841 
842 def clpatch(ui, repo, clname, **opts):
843 	"""import a patch from the code review server
844 
845 	Imports a patch from the code review server into the local client.
846 	If the local client has already modified any of the files that the
847 	patch modifies, this command will refuse to apply the patch.
848 
849 	Submitting an imported patch will keep the original author's
850 	name as the Author: line but add your own name to a Committer: line.
851 	"""
852 	cl, patch, err = DownloadCL(ui, repo, clname)
853 	argv = ["hgpatch"]
854 	if opts["fuzzy"]:
855 		argv += ["--fuzzy"]
856 	if opts["no_incoming"]:
857 		argv += ["--checksync=false"]
858 	if err != "":
859 		return err
860 	try:
861 		cmd = subprocess.Popen(argv, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, close_fds=True)
862 	except:
863 		return "hgpatch: " + ExceptionDetail()
864 	if os.fork() == 0:
865 		cmd.stdin.write(patch)
866 		os._exit(0)
867 	cmd.stdin.close()
868 	out = cmd.stdout.read()
869 	if cmd.wait() != 0 and not opts["ignore_hgpatch_failure"]:
870 		return "hgpatch failed"
871 	cl.local = True
872 	cl.files = out.strip().split()
873 	files = ChangedFiles(ui, repo, [], opts)
874 	extra = Sub(cl.files, files)
875 	if extra:
876 		ui.warn("warning: these files were listed in the patch but not changed:\n\t" + "\n\t".join(extra) + "\n")
877 	cl.Flush(ui, repo)
878 	ui.write(cl.PendingText() + "\n")
879 
880 def download(ui, repo, clname, **opts):
881 	"""download a change from the code review server
882 
883 	Download prints a description of the given change list
884 	followed by its diff, downloaded from the code review server.
885 	"""
886 	cl, patch, err = DownloadCL(ui, repo, clname)
887 	if err != "":
888 		return err
889 	ui.write(cl.EditorText() + "\n")
890 	ui.write(patch + "\n")
891 	return
892 
893 def file(ui, repo, clname, pat, *pats, **opts):
894 	"""assign files to or remove files from a change list
895 
896 	Assign files to or (with -d) remove files from a change list.
897 
898 	The -d option only removes files from the change list.
899 	It does not edit them or remove them from the repository.
900 	"""
901 	pats = tuple([pat] + list(pats))
902 	if not GoodCLName(clname):
903 		return "invalid CL name " + clname
904 
905 	dirty = {}
906 	cl, err = LoadCL(ui, repo, clname, web=False)
907 	if err != '':
908 		return err
909 	if not cl.local:
910 		return "cannot change non-local CL " + clname
911 
912 	files = ChangedFiles(ui, repo, pats, opts)
913 
914 	if opts["delete"]:
915 		oldfiles = Intersect(files, cl.files)
916 		if oldfiles:
917 			if not ui.quiet:
918 				ui.status("# Removing files from CL.  To undo:\n")
919 				ui.status("#	cd %s\n" % (repo.root))
920 				for f in oldfiles:
921 					ui.status("#	hg file %s %s\n" % (cl.name, f))
922 			cl.files = Sub(cl.files, oldfiles)
923 			cl.Flush(ui, repo)
924 		else:
925 			ui.status("no such files in CL")
926 		return
927 
928 	if not files:
929 		return "no such modified files"
930 
931 	files = Sub(files, cl.files)
932 	taken = Taken(ui, repo)
933 	warned = False
934 	for f in files:
935 		if f in taken:
936 			if not warned and not ui.quiet:
937 				ui.status("# Taking files from other CLs.  To undo:\n")
938 				ui.status("#	cd %s\n" % (repo.root))
939 				warned = True
940 			ocl = taken[f]
941 			if not ui.quiet:
942 				ui.status("#	hg file %s %s\n" % (ocl.name, f))
943 			if ocl not in dirty:
944 				ocl.files = Sub(ocl.files, files)
945 				dirty[ocl] = True
946 	cl.files = Add(cl.files, files)
947 	dirty[cl] = True
948 	for d, _ in dirty.items():
949 		d.Flush(ui, repo)
950 	return
951 
952 def gofmt(ui, repo, *pats, **opts):
953 	"""apply gofmt to modified files
954 
955 	Applies gofmt to the modified files in the repository that match
956 	the given patterns.
957 	"""
958 	files = ChangedExistingFiles(ui, repo, pats, opts)
959 	files = [f for f in files if f.endswith(".go")]
960 	if not files:
961 		return "no modified go files"
962 	cwd = os.getcwd()
963 	files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
964 	try:
965 		cmd = ["gofmt", "-l"]
966 		if not opts["list"]:
967 			cmd += ["-w"]
968 		if os.spawnvp(os.P_WAIT, "gofmt", cmd + files) != 0:
969 			raise util.Abort("gofmt did not exit cleanly")
970 	except error.Abort, e:
971 		raise
972 	except:
973 		raise util.Abort("gofmt: " + ExceptionDetail())
974 	return
975 
976 def mail(ui, repo, *pats, **opts):
977 	"""mail a change for review
978 
979 	Uploads a patch to the code review server and then sends mail
980 	to the reviewer and CC list asking for a review.
981 	"""
982 	cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
983 	if err != "":
984 		return err
985 	cl.Upload(ui, repo, gofmt_just_warn=True)
986 	if not cl.reviewer:
987 		# If no reviewer is listed, assign the review to defaultcc.
988 		# This makes sure that it appears in the
989 		# codereview.appspot.com/user/defaultcc
990 		# page, so that it doesn't get dropped on the floor.
991 		if not defaultcc:
992 			return "no reviewers listed in CL"
993 		cl.cc = Sub(cl.cc, defaultcc)
994 		cl.reviewer = defaultcc
995 		cl.Flush(ui, repo)
996 	cl.Mail(ui, repo)
997 
998 def nocommit(ui, repo, *pats, **opts):
999 	"""(disabled when using this extension)"""
1000 	return "The codereview extension is enabled; do not use commit."
1001 
1002 def pending(ui, repo, *pats, **opts):
1003 	"""show pending changes
1004 
1005 	Lists pending changes followed by a list of unassigned but modified files.
1006 	"""
1007 	m = LoadAllCL(ui, repo, web=True)
1008 	names = m.keys()
1009 	names.sort()
1010 	for name in names:
1011 		cl = m[name]
1012 		ui.write(cl.PendingText() + "\n")
1013 
1014 	files = DefaultFiles(ui, repo, [], opts)
1015 	if len(files) > 0:
1016 		s = "Changed files not in any CL:\n"
1017 		for f in files:
1018 			s += "\t" + f + "\n"
1019 		ui.write(s)
1020 
1021 def reposetup(ui, repo):
1022 	global original_match
1023 	if original_match is None:
1024 		original_match = cmdutil.match
1025 		cmdutil.match = ReplacementForCmdutilMatch
1026 		RietveldSetup(ui, repo)
1027 
1028 def CheckContributor(ui, repo, user=None):
1029 	if not user:
1030 		user = ui.config("ui", "username")
1031 		if not user:
1032 			raise util.Abort("[ui] username is not configured in .hgrc")
1033 	_, userline = FindContributor(ui, repo, user, warn=False)
1034 	if not userline:
1035 		raise util.Abort("cannot find %s in CONTRIBUTORS" % (user,))
1036 	return userline
1037 
1038 def FindContributor(ui, repo, user, warn=True):
1039 	m = re.match(r".*<(.*)>", user)
1040 	if m:
1041 		user = m.group(1).lower()
1042 
1043 	if user not in contributors:
1044 		if warn:
1045 			ui.warn("warning: cannot find %s in CONTRIBUTORS\n" % (user,))
1046 		return None, None
1047 	
1048 	user, email = contributors[user]
1049 	return email, "%s <%s>" % (user, email)
1050 
1051 def submit(ui, repo, *pats, **opts):
1052 	"""submit change to remote repository
1053 
1054 	Submits change to remote repository.
1055 	Bails out if the local repository is not in sync with the remote one.
1056 	"""
1057 	repo.ui.quiet = True
1058 	if not opts["no_incoming"] and Incoming(ui, repo, opts):
1059 		return "local repository out of date; must sync before submit"
1060 
1061 	cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
1062 	if err != "":
1063 		return err
1064 
1065 	user = None
1066 	if cl.copied_from:
1067 		user = cl.copied_from
1068 	userline = CheckContributor(ui, repo, user)
1069 
1070 	about = ""
1071 	if cl.reviewer:
1072 		about += "R=" + JoinComma([CutDomain(s) for s in cl.reviewer]) + "\n"
1073 	if opts.get('tbr'):
1074 		tbr = SplitCommaSpace(opts.get('tbr'))
1075 		cl.reviewer = Add(cl.reviewer, tbr)
1076 		about += "TBR=" + JoinComma([CutDomain(s) for s in tbr]) + "\n"
1077 	if cl.cc:
1078 		about += "CC=" + JoinComma([CutDomain(s) for s in cl.cc]) + "\n"
1079 
1080 	if not cl.reviewer:
1081 		return "no reviewers listed in CL"
1082 
1083 	if not cl.local:
1084 		return "cannot submit non-local CL"
1085 
1086 	# upload, to sync current patch and also get change number if CL is new.
1087 	if not cl.copied_from:
1088 		cl.Upload(ui, repo, gofmt_just_warn=True)
1089 
1090 	# check gofmt for real; allowed upload to warn in order to save CL.
1091 	cl.Flush(ui, repo)
1092 	CheckGofmt(ui, repo, cl.files)
1093 
1094 	about += "%s%s\n" % (server_url_base, cl.name)
1095 
1096 	if cl.copied_from:
1097 		about += "\nCommitter: " + CheckContributor(ui, repo, None) + "\n"
1098 
1099 	if not cl.mailed and not cl.copied_from:		# in case this is TBR
1100 		cl.Mail(ui, repo)
1101 
1102 	# submit changes locally
1103 	date = opts.get('date')
1104 	if date:
1105 		opts['date'] = util.parsedate(date)
1106 	opts['message'] = cl.desc.rstrip() + "\n\n" + about
1107 
1108 	if opts['dryrun']:
1109 		print "NOT SUBMITTING:"
1110 		print "User: ", userline
1111 		print "Message:"
1112 		print Indent(opts['message'], "\t")
1113 		print "Files:"
1114 		print Indent('\n'.join(cl.files), "\t")
1115 		return "dry run; not submitted"
1116 
1117 	m = match.exact(repo.root, repo.getcwd(), cl.files)
1118 	node = repo.commit(opts['message'], userline, opts.get('date'), m)
1119 	if not node:
1120 		return "nothing changed"
1121 
1122 	# push to remote; if it fails for any reason, roll back
1123 	try:
1124 		log = repo.changelog
1125 		rev = log.rev(node)
1126 		parents = log.parentrevs(rev)
1127 		if (rev-1 not in parents and
1128 				(parents == (nullrev, nullrev) or
1129 				len(log.heads(log.node(parents[0]))) > 1 and
1130 				(parents[1] == nullrev or len(log.heads(log.node(parents[1]))) > 1))):
1131 			# created new head
1132 			raise util.Abort("local repository out of date; must sync before submit")
1133 
1134 		# push changes to remote.
1135 		# if it works, we're committed.
1136 		# if not, roll back
1137 		other = getremote(ui, repo, opts)
1138 		r = repo.push(other, False, None)
1139 		if r == 0:
1140 			raise util.Abort("local repository out of date; must sync before submit")
1141 	except:
1142 		repo.rollback()
1143 		raise
1144 
1145 	# we're committed. upload final patch, close review, add commit message
1146 	changeURL = short(node)
1147 	url = other.url()
1148 	m = re.match("^https?://([^@/]+@)?([^.]+)\.googlecode\.com/hg/", url)
1149 	if m:
1150 		changeURL = "http://code.google.com/p/%s/source/detail?r=%s" % (m.group(2), changeURL)
1151 	else:
1152 		print >>sys.stderr, "URL: ", url
1153 	pmsg = "*** Submitted as " + changeURL + " ***\n\n" + opts['message']
1154 
1155 	# When posting, move reviewers to CC line,
1156 	# so that the issue stops showing up in their "My Issues" page.
1157 	PostMessage(ui, cl.name, pmsg, reviewers="", cc=JoinComma(cl.reviewer+cl.cc))
1158 
1159 	if not cl.copied_from:
1160 		EditDesc(cl.name, closed="checked")
1161 	cl.Delete(ui, repo)
1162 
1163 def sync(ui, repo, **opts):
1164 	"""synchronize with remote repository
1165 
1166 	Incorporates recent changes from the remote repository
1167 	into the local repository.
1168 	"""
1169 	if not opts["local"]:
1170 		ui.status = sync_note
1171 		ui.note = sync_note
1172 		other = getremote(ui, repo, opts)
1173 		modheads = repo.pull(other)
1174 		err = commands.postincoming(ui, repo, modheads, True, "tip")
1175 		if err:
1176 			return err
1177 	commands.update(ui, repo)
1178 	sync_changes(ui, repo)
1179 
1180 def sync_note(msg):
1181 	# we run sync (pull -u) in verbose mode to get the
1182 	# list of files being updated, but that drags along
1183 	# a bunch of messages we don't care about.
1184 	# omit them.
1185 	if msg == 'resolving manifests\n':
1186 		return
1187 	if msg == 'searching for changes\n':
1188 		return
1189 	if msg == "couldn't find merge tool hgmerge\n":
1190 		return
1191 	sys.stdout.write(msg)
1192 
1193 def sync_changes(ui, repo):
1194 	# Look through recent change log descriptions to find
1195 	# potential references to http://.*/our-CL-number.
1196 	# Double-check them by looking at the Rietveld log.
1197 	def Rev(rev):
1198 		desc = repo[rev].description().strip()
1199 		for clname in re.findall('(?m)^http://(?:[^\n]+)/([0-9]+)$', desc):
1200 			if IsLocalCL(ui, repo, clname) and IsRietveldSubmitted(ui, clname, repo[rev].hex()):
1201 				ui.warn("CL %s submitted as %s; closing\n" % (clname, repo[rev]))
1202 				cl, err = LoadCL(ui, repo, clname, web=False)
1203 				if err != "":
1204 					ui.warn("loading CL %s: %s\n" % (clname, err))
1205 					continue
1206 				if not cl.copied_from:
1207 					EditDesc(cl.name, closed="checked")
1208 				cl.Delete(ui, repo)
1209 
1210 	if hgversion < '1.4':
1211 		get = util.cachefunc(lambda r: repo[r].changeset())
1212 		changeiter, matchfn = cmdutil.walkchangerevs(ui, repo, [], get, {'rev': None})
1213 		n = 0
1214 		for st, rev, fns in changeiter:
1215 			if st != 'iter':
1216 				continue
1217 			n += 1
1218 			if n > 100:
1219 				break
1220 			Rev(rev)
1221 	else:
1222 		matchfn = cmdutil.match(repo, [], {'rev': None})
1223 		def prep(ctx, fns):
1224 			pass
1225 		for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev': None}, prep):
1226 			Rev(ctx.rev())
1227 
1228 	# Remove files that are not modified from the CLs in which they appear.
1229 	all = LoadAllCL(ui, repo, web=False)
1230 	changed = ChangedFiles(ui, repo, [], {})
1231 	for _, cl in all.items():
1232 		extra = Sub(cl.files, changed)
1233 		if extra:
1234 			ui.warn("Removing unmodified files from CL %s:\n" % (cl.name,))
1235 			for f in extra:
1236 				ui.warn("\t%s\n" % (f,))
1237 			cl.files = Sub(cl.files, extra)
1238 			cl.Flush(ui, repo)
1239 		if not cl.files:
1240 			ui.warn("CL %s has no files; suggest hg change -d %s\n" % (cl.name, cl.name))
1241 	return
1242 
1243 def uisetup(ui):
1244 	if "^commit|ci" in commands.table:
1245 		commands.table["^commit|ci"] = (nocommit, [], "")
1246 
1247 def upload(ui, repo, name, **opts):
1248 	"""upload diffs to the code review server
1249 
1250 	Uploads the current modifications for a given change to the server.
1251 	"""
1252 	repo.ui.quiet = True
1253 	cl, err = LoadCL(ui, repo, name, web=True)
1254 	if err != "":
1255 		return err
1256 	if not cl.local:
1257 		return "cannot upload non-local change"
1258 	cl.Upload(ui, repo)
1259 	print "%s%s\n" % (server_url_base, cl.name)
1260 	return
1261 
1262 review_opts = [
1263 	('r', 'reviewer', '', 'add reviewer'),
1264 	('', 'cc', '', 'add cc'),
1265 	('', 'tbr', '', 'add future reviewer'),
1266 	('m', 'message', '', 'change description (for new change)'),
1267 ]
1268 
1269 cmdtable = {
1270 	# The ^ means to show this command in the help text that
1271 	# is printed when running hg with no arguments.
1272 	"^change": (
1273 		change,
1274 		[
1275 			('d', 'delete', None, 'delete existing change list'),
1276 			('D', 'deletelocal', None, 'delete locally, but do not change CL on server'),
1277 			('i', 'stdin', None, 'read change list from standard input'),
1278 			('o', 'stdout', None, 'print change list to standard output'),
1279 		],
1280 		"[-d | -D] [-i] [-o] change# or FILE ..."
1281 	),
1282 	"^clpatch": (
1283 		clpatch,
1284 		[
1285 			('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
1286 			('', 'no_incoming', None, 'disable check for incoming changes'),
1287 			('', 'fuzzy', None, 'attempt to adjust patch line numbers'),
1288 		],
1289 		"change#"
1290 	),
1291 	# Would prefer to call this codereview-login, but then
1292 	# hg help codereview prints the help for this command
1293 	# instead of the help for the extension.
1294 	"code-login": (
1295 		code_login,
1296 		[],
1297 		"",
1298 	),
1299 	"commit|ci": (
1300 		nocommit,
1301 		[],
1302 		"",
1303 	),
1304 	"^download": (
1305 		download,
1306 		[],
1307 		"change#"
1308 	),
1309 	"^file": (
1310 		file,
1311 		[
1312 			('d', 'delete', None, 'delete files from change list (but not repository)'),
1313 		],
1314 		"[-d] change# FILE ..."
1315 	),
1316 	"^gofmt": (
1317 		gofmt,
1318 		[
1319 			('l', 'list', None, 'list files that would change, but do not edit them'),
1320 		],
1321 		"FILE ..."
1322 	),
1323 	"^pending|p": (
1324 		pending,
1325 		[],
1326 		"[FILE ...]"
1327 	),
1328 	"^mail": (
1329 		mail,
1330 		review_opts + [
1331 		] + commands.walkopts,
1332 		"[-r reviewer] [--cc cc] [change# | file ...]"
1333 	),
1334 	"^submit": (
1335 		submit,
1336 		review_opts + [
1337 			('', 'no_incoming', None, 'disable initial incoming check (for testing)'),
1338 			('n', 'dryrun', None, 'make change only locally (for testing)'),
1339 		] + commands.walkopts + commands.commitopts + commands.commitopts2,
1340 		"[-r reviewer] [--cc cc] [change# | file ...]"
1341 	),
1342 	"^sync": (
1343 		sync,
1344 		[
1345 			('', 'local', None, 'do not pull changes from remote repository')
1346 		],
1347 		"[--local]",
1348 	),
1349 	"^upload": (
1350 		upload,
1351 		[],
1352 		"change#"
1353 	),
1354 }
1355 
1356 
1357 #######################################################################
1358 # Wrappers around upload.py for interacting with Rietveld
1359 
1360 # HTML form parser
1361 class FormParser(HTMLParser):
1362 	def __init__(self):
1363 		self.map = {}
1364 		self.curtag = None
1365 		self.curdata = None
1366 		HTMLParser.__init__(self)
1367 	def handle_starttag(self, tag, attrs):
1368 		if tag == "input":
1369 			key = None
1370 			value = ''
1371 			for a in attrs:
1372 				if a[0] == 'name':
1373 					key = a[1]
1374 				if a[0] == 'value':
1375 					value = a[1]
1376 			if key is not None:
1377 				self.map[key] = value
1378 		if tag == "textarea":
1379 			key = None
1380 			for a in attrs:
1381 				if a[0] == 'name':
1382 					key = a[1]
1383 			if key is not None:
1384 				self.curtag = key
1385 				self.curdata = ''
1386 	def handle_endtag(self, tag):
1387 		if tag == "textarea" and self.curtag is not None:
1388 			self.map[self.curtag] = self.curdata
1389 			self.curtag = None
1390 			self.curdata = None
1391 	def handle_charref(self, name):
1392 		self.handle_data(unichr(int(name)))
1393 	def handle_entityref(self, name):
1394 		import htmlentitydefs
1395 		if name in htmlentitydefs.entitydefs:
1396 			self.handle_data(htmlentitydefs.entitydefs[name])
1397 		else:
1398 			self.handle_data("&" + name + ";")
1399 	def handle_data(self, data):
1400 		if self.curdata is not None:
1401 			self.curdata += data.decode("utf-8").encode("utf-8")
1402 
1403 # XML parser
1404 def XMLGet(ui, path):
1405 	try:
1406 		data = MySend(path, force_auth=False);
1407 	except:
1408 		ui.warn("XMLGet %s: %s\n" % (path, ExceptionDetail()))
1409 		return None
1410 	return ET.XML(data)
1411 
1412 def IsRietveldSubmitted(ui, clname, hex):
1413 	feed = XMLGet(ui, "/rss/issue/" + clname)
1414 	if feed is None:
1415 		return False
1416 	for sum in feed.findall("{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}summary"):
1417 		text = sum.findtext("", None).strip()
1418 		m = re.match('\*\*\* Submitted as [^*]*?([0-9a-f]+) \*\*\*', text)
1419 		if m is not None and len(m.group(1)) >= 8 and hex.startswith(m.group(1)):
1420 			return True
1421 	return False
1422 
1423 def DownloadCL(ui, repo, clname):
1424 	cl, err = LoadCL(ui, repo, clname)
1425 	if err != "":
1426 		return None, None, "error loading CL %s: %s" % (clname, ExceptionDetail())
1427 
1428 	# Grab RSS feed to learn about CL
1429 	feed = XMLGet(ui, "/rss/issue/" + clname)
1430 	if feed is None:
1431 		return None, None, "cannot download CL"
1432 
1433 	# Find most recent diff
1434 	diff = None
1435 	prefix = 'http://' + server + '/'
1436 	for link in feed.findall("{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}link"):
1437 		if link.get('rel') != 'alternate':
1438 			continue
1439 		text = link.get('href')
1440 		if not text.startswith(prefix) or not text.endswith('.diff'):
1441 			continue
1442 		diff = text[len(prefix)-1:]
1443 	if diff is None:
1444 		return None, None, "CL has no diff"
1445 	diffdata = MySend(diff, force_auth=False)
1446 
1447 	# Find author - first entry will be author who created CL.
1448 	nick = None
1449 	for author in feed.findall("{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}author/{http://www.w3.org/2005/Atom}name"):
1450 		nick = author.findtext("", None).strip()
1451 		break
1452 	if not nick:
1453 		return None, None, "CL has no author"
1454 
1455 	# The author is just a nickname: get the real email address.
1456 	try:
1457 		# want URL-encoded nick, but without a=, and rietveld rejects + for %20.
1458 		url = "/user_popup/" + urllib.urlencode({"a": nick})[2:].replace("+", "%20")
1459 		data = MySend(url, force_auth=False)
1460 	except:
1461 		ui.warn("error looking up %s: %s\n" % (nick, ExceptionDetail()))
1462 		cl.copied_from = nick+"@needtofix"
1463 		return cl, diffdata, ""
1464 	match = re.match(r"<b>(.*) \((.*)\)</b>", data)
1465 	if not match:
1466 		return None, None, "error looking up %s: cannot parse result %s" % (nick, repr(data))
1467 	if match.group(1) != nick and match.group(2) != nick:
1468 		return None, None, "error looking up %s: got info for %s, %s" % (nick, match.group(1), match.group(2))
1469 	email = match.group(1)
1470 
1471 	# Print warning if email is not in CONTRIBUTORS file.
1472 	FindContributor(ui, repo, email)
1473 	cl.copied_from = email
1474 
1475 	return cl, diffdata, ""
1476 
1477 def MySend(request_path, payload=None,
1478            content_type="application/octet-stream",
1479            timeout=None, force_auth=True,
1480            **kwargs):
1481      """Run MySend1 maybe twice, because Rietveld is unreliable."""
1482      try:
1483          return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
1484      except Exception, e:
1485          if type(e) == urllib2.HTTPError and e.code == 403:	# forbidden, it happens
1486          	raise
1487          print >>sys.stderr, "Loading "+request_path+": "+ExceptionDetail()+"; trying again in 2 seconds."
1488      time.sleep(2)
1489      return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
1490 
1491 
1492 # Like upload.py Send but only authenticates when the
1493 # redirect is to www.google.com/accounts.  This keeps
1494 # unnecessary redirects from happening during testing.
1495 def MySend1(request_path, payload=None,
1496            content_type="application/octet-stream",
1497            timeout=None, force_auth=True,
1498            **kwargs):
1499     """Sends an RPC and returns the response.
1500 
1501     Args:
1502       request_path: The path to send the request to, eg /api/appversion/create.
1503       payload: The body of the request, or None to send an empty request.
1504       content_type: The Content-Type header to use.
1505       timeout: timeout in seconds; default None i.e. no timeout.
1506         (Note: for large requests on OS X, the timeout doesn't work right.)
1507       kwargs: Any keyword arguments are converted into query string parameters.
1508 
1509     Returns:
1510       The response body, as a string.
1511     """
1512     # TODO: Don't require authentication.  Let the server say
1513     # whether it is necessary.
1514     global rpc
1515     if rpc == None:
1516     	rpc = GetRpcServer(upload_options)
1517     self = rpc
1518     if not self.authenticated and force_auth:
1519       self._Authenticate()
1520     if request_path is None:
1521       return
1522 
1523     old_timeout = socket.getdefaulttimeout()
1524     socket.setdefaulttimeout(timeout)
1525     try:
1526       tries = 0
1527       while True:
1528         tries += 1
1529         args = dict(kwargs)
1530         url = "http://%s%s" % (self.host, request_path)
1531         if args:
1532           url += "?" + urllib.urlencode(args)
1533         req = self._CreateRequest(url=url, data=payload)
1534         req.add_header("Content-Type", content_type)
1535         try:
1536           f = self.opener.open(req)
1537           response = f.read()
1538           f.close()
1539           # Translate \r\n into \n, because Rietveld doesn't.
1540           response = response.replace('\r\n', '\n')
1541           return response
1542         except urllib2.HTTPError, e:
1543           if tries > 3:
1544             raise
1545           elif e.code == 401:
1546             self._Authenticate()
1547           elif e.code == 302:
1548             loc = e.info()["location"]
1549             if not loc.startswith('https://www.google.com/a') or loc.find('/ServiceLogin') < 0:
1550               return ''
1551             self._Authenticate()
1552           else:
1553             raise
1554     finally:
1555       socket.setdefaulttimeout(old_timeout)
1556 
1557 def GetForm(url):
1558 	f = FormParser()
1559 	f.feed(MySend(url))
1560 	f.close()
1561 	for k,v in f.map.items():
1562 		f.map[k] = v.replace("\r\n", "\n");
1563 	return f.map
1564 
1565 # Fetch the settings for the CL, like reviewer and CC list, by
1566 # scraping the Rietveld editing forms.
1567 def GetSettings(issue):
1568 	# The /issue/edit page has everything but only the
1569 	# CL owner is allowed to fetch it (and submit it).
1570 	f = None
1571 	try:
1572 		f = GetForm("/" + issue + "/edit")
1573 	except:
1574 		pass
1575 	if not f or 'reviewers' not in f:
1576 		# Maybe we're not the CL owner.  Fall back to the
1577 		# /publish page, which has the reviewer and CC lists,
1578 		# and then fetch the description separately.
1579 		f = GetForm("/" + issue + "/publish")
1580 		f['description'] = MySend("/"+issue+"/description", force_auth=False)
1581 	return f
1582 
1583 def EditDesc(issue, subject=None, desc=None, reviewers=None, cc=None, closed=None):
1584 	form_fields = GetForm("/" + issue + "/edit")
1585 	if subject is not None:
1586 		form_fields['subject'] = subject
1587 	if desc is not None:
1588 		form_fields['description'] = desc
1589 	if reviewers is not None:
1590 		form_fields['reviewers'] = reviewers
1591 	if cc is not None:
1592 		form_fields['cc'] = cc
1593 	if closed is not None:
1594 		form_fields['closed'] = closed
1595 	ctype, body = EncodeMultipartFormData(form_fields.items(), [])
1596 	response = MySend("/" + issue + "/edit", body, content_type=ctype)
1597 	if response != "":
1598 		print >>sys.stderr, "Error editing description:\n" + "Sent form: \n", form_fields, "\n", response
1599 		sys.exit(2)
1600 
1601 def PostMessage(ui, issue, message, reviewers=None, cc=None, send_mail=True, subject=None):
1602 	form_fields = GetForm("/" + issue + "/publish")
1603 	if reviewers is not None:
1604 		form_fields['reviewers'] = reviewers
1605 	if cc is not None:
1606 		form_fields['cc'] = cc
1607 	if send_mail:
1608 		form_fields['send_mail'] = "checked"
1609 	else:
1610 		del form_fields['send_mail']
1611 	if subject is not None:
1612 		form_fields['subject'] = subject
1613 	form_fields['message'] = message
1614 
1615 	form_fields['message_only'] = '1'	# Don't include draft comments
1616 	if reviewers is not None or cc is not None:
1617 		form_fields['message_only'] = ''	# Must set '' in order to override cc/reviewer
1618 	ctype = "applications/x-www-form-urlencoded"
1619 	body = urllib.urlencode(form_fields)
1620 	response = MySend("/" + issue + "/publish", body, content_type=ctype)
1621 	if response != "":
1622 		print response
1623 		sys.exit(2)
1624 
1625 class opt(object):
1626 	pass
1627 
1628 def RietveldSetup(ui, repo):
1629 	global defaultcc, upload_options, rpc, server, server_url_base, force_google_account, verbosity, contributors
1630 
1631 	# Read repository-specific options from lib/codereview/codereview.cfg
1632 	try:
1633 		f = open(repo.root + '/lib/codereview/codereview.cfg')
1634 		for line in f:
1635 			if line.startswith('defaultcc: '):
1636 				defaultcc = SplitCommaSpace(line[10:])
1637 	except:
1638 		# If there are no options, chances are good this is not
1639 		# a code review repository; stop now before we foul
1640 		# things up even worse.  Might also be that repo doesn't
1641 		# even have a root.  See issue 959.
1642 		return
1643 
1644 	try:
1645 		f = open(repo.root + '/CONTRIBUTORS', 'r')
1646 	except:
1647 		raise util.Abort("cannot open %s: %s" % (repo.root+'/CONTRIBUTORS', ExceptionDetail()))
1648 	for line in f:
1649 		# CONTRIBUTORS is a list of lines like:
1650 		#	Person <email>
1651 		#	Person <email> <alt-email>
1652 		# The first email address is the one used in commit logs.
1653 		if line.startswith('#'):
1654 			continue
1655 		m = re.match(r"([^<>]+\S)\s+(<[^<>\s]+>)((\s+<[^<>\s]+>)*)\s*$", line)
1656 		if m:
1657 			name = m.group(1)
1658 			email = m.group(2)[1:-1]
1659 			contributors[email.lower()] = (name, email)
1660 			for extra in m.group(3).split():
1661 				contributors[extra[1:-1].lower()] = (name, email)
1662 	
1663 
1664 	# TODO(rsc): If the repository config has no codereview section,
1665 	# do not enable the extension.  This allows users to
1666 	# put the extension in their global .hgrc but only
1667 	# enable it for some repositories.
1668 	# if not ui.has_section("codereview"):
1669 	# 	cmdtable = {}
1670 	# 	return
1671 
1672 	if not ui.verbose:
1673 		verbosity = 0
1674 
1675 	# Config options.
1676 	x = ui.config("codereview", "server")
1677 	if x is not None:
1678 		server = x
1679 
1680 	# TODO(rsc): Take from ui.username?
1681 	email = None
1682 	x = ui.config("codereview", "email")
1683 	if x is not None:
1684 		email = x
1685 
1686 	server_url_base = "http://" + server + "/"
1687 
1688 	testing = ui.config("codereview", "testing")
1689 	force_google_account = ui.configbool("codereview", "force_google_account", False)
1690 
1691 	upload_options = opt()
1692 	upload_options.email = email
1693 	upload_options.host = None
1694 	upload_options.verbose = 0
1695 	upload_options.description = None
1696 	upload_options.description_file = None
1697 	upload_options.reviewers = None
1698 	upload_options.cc = None
1699 	upload_options.message = None
1700 	upload_options.issue = None
1701 	upload_options.download_base = False
1702 	upload_options.revision = None
1703 	upload_options.send_mail = False
1704 	upload_options.vcs = None
1705 	upload_options.server = server
1706 	upload_options.save_cookies = True
1707 
1708 	if testing:
1709 		upload_options.save_cookies = False
1710 		upload_options.email = "test@example.com"
1711 
1712 	rpc = None
1713 
1714 #######################################################################
1715 # We keep a full copy of upload.py here to avoid import path hell.
1716 # It would be nice if hg added the hg repository root
1717 # to the default PYTHONPATH.
1718 
1719 # Edit .+2,<hget http://codereview.appspot.com/static/upload.py
1720 
1721 #!/usr/bin/env python
1722 #
1723 # Copyright 2007 Google Inc.
1724 #
1725 # Licensed under the Apache License, Version 2.0 (the "License");
1726 # you may not use this file except in compliance with the License.
1727 # You may obtain a copy of the License at
1728 #
1729 #     http://www.apache.org/licenses/LICENSE-2.0
1730 #
1731 # Unless required by applicable law or agreed to in writing, software
1732 # distributed under the License is distributed on an "AS IS" BASIS,
1733 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1734 # See the License for the specific language governing permissions and
1735 # limitations under the License.
1736 
1737 """Tool for uploading diffs from a version control system to the codereview app.
1738 
1739 Usage summary: upload.py [options] [-- diff_options]
1740 
1741 Diff options are passed to the diff command of the underlying system.
1742 
1743 Supported version control systems:
1744   Git
1745   Mercurial
1746   Subversion
1747 
1748 It is important for Git/Mercurial users to specify a tree/node/branch to diff
1749 against by using the '--rev' option.
1750 """
1751 # This code is derived from appcfg.py in the App Engine SDK (open source),
1752 # and from ASPN recipe #146306.
1753 
1754 import cookielib
1755 import getpass
1756 import logging
1757 import mimetypes
1758 import optparse
1759 import os
1760 import re
1761 import socket
1762 import subprocess
1763 import sys
1764 import urllib
1765 import urllib2
1766 import urlparse
1767 
1768 # The md5 module was deprecated in Python 2.5.
1769 try:
1770   from hashlib import md5
1771 except ImportError:
1772   from md5 import md5
1773 
1774 try:
1775   import readline
1776 except ImportError:
1777   pass
1778 
1779 # The logging verbosity:
1780 #  0: Errors only.
1781 #  1: Status messages.
1782 #  2: Info logs.
1783 #  3: Debug logs.
1784 verbosity = 1
1785 
1786 # Max size of patch or base file.
1787 MAX_UPLOAD_SIZE = 900 * 1024
1788 
1789 # Constants for version control names.  Used by GuessVCSName.
1790 VCS_GIT = "Git"
1791 VCS_MERCURIAL = "Mercurial"
1792 VCS_SUBVERSION = "Subversion"
1793 VCS_UNKNOWN = "Unknown"
1794 
1795 # whitelist for non-binary filetypes which do not start with "text/"
1796 # .mm (Objective-C) shows up as application/x-freemind on my Linux box.
1797 TEXT_MIMETYPES = ['application/javascript', 'application/x-javascript',
1798                   'application/x-freemind']
1799 
1800 VCS_ABBREVIATIONS = {
1801   VCS_MERCURIAL.lower(): VCS_MERCURIAL,
1802   "hg": VCS_MERCURIAL,
1803   VCS_SUBVERSION.lower(): VCS_SUBVERSION,
1804   "svn": VCS_SUBVERSION,
1805   VCS_GIT.lower(): VCS_GIT,
1806 }
1807 
1808 
1809 def GetEmail(prompt):
1810   """Prompts the user for their email address and returns it.
1811 
1812   The last used email address is saved to a file and offered up as a suggestion
1813   to the user. If the user presses enter without typing in anything the last
1814   used email address is used. If the user enters a new address, it is saved
1815   for next time we prompt.
1816 
1817   """
1818   last_email_file_name = os.path.expanduser("~/.last_codereview_email_address")
1819   last_email = ""
1820   if os.path.exists(last_email_file_name):
1821     try:
1822       last_email_file = open(last_email_file_name, "r")
1823       last_email = last_email_file.readline().strip("\n")
1824       last_email_file.close()
1825       prompt += " [%s]" % last_email
1826     except IOError, e:
1827       pass
1828   email = raw_input(prompt + ": ").strip()
1829   if email:
1830     try:
1831       last_email_file = open(last_email_file_name, "w")
1832       last_email_file.write(email)
1833       last_email_file.close()
1834     except IOError, e:
1835       pass
1836   else:
1837     email = last_email
1838   return email
1839 
1840 
1841 def StatusUpdate(msg):
1842   """Print a status message to stdout.
1843 
1844   If 'verbosity' is greater than 0, print the message.
1845 
1846   Args:
1847     msg: The string to print.
1848   """
1849   if verbosity > 0:
1850     print msg
1851 
1852 
1853 def ErrorExit(msg):
1854   """Print an error message to stderr and exit."""
1855   print >>sys.stderr, msg
1856   sys.exit(1)
1857 
1858 
1859 class ClientLoginError(urllib2.HTTPError):
1860   """Raised to indicate there was an error authenticating with ClientLogin."""
1861 
1862   def __init__(self, url, code, msg, headers, args):
1863     urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
1864     self.args = args
1865     self.reason = args["Error"]
1866 
1867 
1868 class AbstractRpcServer(object):
1869   """Provides a common interface for a simple RPC server."""
1870 
1871   def __init__(self, host, auth_function, host_override=None, extra_headers={},
1872                save_cookies=False):
1873     """Creates a new HttpRpcServer.
1874 
1875     Args:
1876       host: The host to send requests to.
1877       auth_function: A function that takes no arguments and returns an
1878         (email, password) tuple when called. Will be called if authentication
1879         is required.
1880       host_override: The host header to send to the server (defaults to host).
1881       extra_headers: A dict of extra headers to append to every request.
1882       save_cookies: If True, save the authentication cookies to local disk.
1883         If False, use an in-memory cookiejar instead.  Subclasses must
1884         implement this functionality.  Defaults to False.
1885     """
1886     self.host = host
1887     self.host_override = host_override
1888     self.auth_function = auth_function
1889     self.authenticated = False
1890     self.extra_headers = extra_headers
1891     self.save_cookies = save_cookies
1892     self.opener = self._GetOpener()
1893     if self.host_override:
1894       logging.info("Server: %s; Host: %s", self.host, self.host_override)
1895     else:
1896       logging.info("Server: %s", self.host)
1897 
1898   def _GetOpener(self):
1899     """Returns an OpenerDirector for making HTTP requests.
1900 
1901     Returns:
1902       A urllib2.OpenerDirector object.
1903     """
1904     raise NotImplementedError()
1905 
1906   def _CreateRequest(self, url, data=None):
1907     """Creates a new urllib request."""
1908     logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
1909     req = urllib2.Request(url, data=data)
1910     if self.host_override:
1911       req.add_header("Host", self.host_override)
1912     for key, value in self.extra_headers.iteritems():
1913       req.add_header(key, value)
1914     return req
1915 
1916   def _GetAuthToken(self, email, password):
1917     """Uses ClientLogin to authenticate the user, returning an auth token.
1918 
1919     Args:
1920       email:    The user's email address
1921       password: The user's password
1922 
1923     Raises:
1924       ClientLoginError: If there was an error authenticating with ClientLogin.
1925       HTTPError: If there was some other form of HTTP error.
1926 
1927     Returns:
1928       The authentication token returned by ClientLogin.
1929     """
1930     account_type = "GOOGLE"
1931     if self.host.endswith(".google.com") and not force_google_account:
1932       # Needed for use inside Google.
1933       account_type = "HOSTED"
1934     req = self._CreateRequest(
1935         url="https://www.google.com/accounts/ClientLogin",
1936         data=urllib.urlencode({
1937             "Email": email,
1938             "Passwd": password,
1939             "service": "ah",
1940             "source": "rietveld-codereview-upload",
1941             "accountType": account_type,
1942         }),
1943     )
1944     try:
1945       response = self.opener.open(req)
1946       response_body = response.read()
1947       response_dict = dict(x.split("=")
1948                            for x in response_body.split("\n") if x)
1949       return response_dict["Auth"]
1950     except urllib2.HTTPError, e:
1951       if e.code == 403:
1952         body = e.read()
1953         response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
1954         raise ClientLoginError(req.get_full_url(), e.code, e.msg,
1955                                e.headers, response_dict)
1956       else:
1957         raise
1958 
1959   def _GetAuthCookie(self, auth_token):
1960     """Fetches authentication cookies for an authentication token.
1961 
1962     Args:
1963       auth_token: The authentication token returned by ClientLogin.
1964 
1965     Raises:
1966       HTTPError: If there was an error fetching the authentication cookies.
1967     """
1968     # This is a dummy value to allow us to identify when we're successful.
1969     continue_location = "http://localhost/"
1970     args = {"continue": continue_location, "auth": auth_token}
1971     req = self._CreateRequest("http://%s/_ah/login?%s" %
1972                               (self.host, urllib.urlencode(args)))
1973     try:
1974       response = self.opener.open(req)
1975     except urllib2.HTTPError, e:
1976       response = e
1977     if (response.code != 302 or
1978         response.info()["location"] != continue_location):
1979       raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,
1980                               response.headers, response.fp)
1981     self.authenticated = True
1982 
1983   def _Authenticate(self):
1984     """Authenticates the user.
1985 
1986     The authentication process works as follows:
1987      1) We get a username and password from the user
1988      2) We use ClientLogin to obtain an AUTH token for the user
1989         (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
1990      3) We pass the auth token to /_ah/login on the server to obtain an
1991         authentication cookie. If login was successful, it tries to redirect
1992         us to the URL we provided.
1993 
1994     If we attempt to access the upload API without first obtaining an
1995     authentication cookie, it returns a 401 response (or a 302) and
1996     directs us to authenticate ourselves with ClientLogin.
1997     """
1998     for i in range(3):
1999       credentials = self.auth_function()
2000       try:
2001         auth_token = self._GetAuthToken(credentials[0], credentials[1])
2002       except ClientLoginError, e:
2003         if e.reason == "BadAuthentication":
2004           print >>sys.stderr, "Invalid username or password."
2005           continue
2006         if e.reason == "CaptchaRequired":
2007           print >>sys.stderr, (
2008               "Please go to\n"
2009               "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
2010               "and verify you are a human.  Then try again.")
2011           break
2012         if e.reason == "NotVerified":
2013           print >>sys.stderr, "Account not verified."
2014           break
2015         if e.reason == "TermsNotAgreed":
2016           print >>sys.stderr, "User has not agreed to TOS."
2017           break
2018         if e.reason == "AccountDeleted":
2019           print >>sys.stderr, "The user account has been deleted."
2020           break
2021         if e.reason == "AccountDisabled":
2022           print >>sys.stderr, "The user account has been disabled."
2023           break
2024         if e.reason == "ServiceDisabled":
2025           print >>sys.stderr, ("The user's access to the service has been "
2026                                "disabled.")
2027           break
2028         if e.reason == "ServiceUnavailable":
2029           print >>sys.stderr, "The service is not available; try again later."
2030           break
2031         raise
2032       self._GetAuthCookie(auth_token)
2033       return
2034 
2035   def Send(self, request_path, payload=None,
2036            content_type="application/octet-stream",
2037            timeout=None,
2038            **kwargs):
2039     """Sends an RPC and returns the response.
2040 
2041     Args:
2042       request_path: The path to send the request to, eg /api/appversion/create.
2043       payload: The body of the request, or None to send an empty request.
2044       content_type: The Content-Type header to use.
2045       timeout: timeout in seconds; default None i.e. no timeout.
2046         (Note: for large requests on OS X, the timeout doesn't work right.)
2047       kwargs: Any keyword arguments are converted into query string parameters.
2048 
2049     Returns:
2050       The response body, as a string.
2051     """
2052     # TODO: Don't require authentication.  Let the server say
2053     # whether it is necessary.
2054     if not self.authenticated:
2055       self._Authenticate()
2056 
2057     old_timeout = socket.getdefaulttimeout()
2058     socket.setdefaulttimeout(timeout)
2059     try:
2060       tries = 0
2061       while True:
2062         tries += 1
2063         args = dict(kwargs)
2064         url = "http://%s%s" % (self.host, request_path)
2065         if args:
2066           url += "?" + urllib.urlencode(args)
2067         req = self._CreateRequest(url=url, data=payload)
2068         req.add_header("Content-Type", content_type)
2069         try:
2070           f = self.opener.open(req)
2071           response = f.read()
2072           f.close()
2073           return response
2074         except urllib2.HTTPError, e:
2075           if tries > 3:
2076             raise
2077           elif e.code == 401 or e.code == 302:
2078             self._Authenticate()
2079           else:
2080             raise
2081     finally:
2082       socket.setdefaulttimeout(old_timeout)
2083 
2084 
2085 class HttpRpcServer(AbstractRpcServer):
2086   """Provides a simplified RPC-style interface for HTTP requests."""
2087 
2088   def _Authenticate(self):
2089     """Save the cookie jar after authentication."""
2090     super(HttpRpcServer, self)._Authenticate()
2091     if self.save_cookies:
2092       StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)
2093       self.cookie_jar.save()
2094 
2095   def _GetOpener(self):
2096     """Returns an OpenerDirector that supports cookies and ignores redirects.
2097 
2098     Returns:
2099       A urllib2.OpenerDirector object.
2100     """
2101     opener = urllib2.OpenerDirector()
2102     opener.add_handler(urllib2.ProxyHandler())
2103     opener.add_handler(urllib2.UnknownHandler())
2104     opener.add_handler(urllib2.HTTPHandler())
2105     opener.add_handler(urllib2.HTTPDefaultErrorHandler())
2106     opener.add_handler(urllib2.HTTPSHandler())
2107     opener.add_handler(urllib2.HTTPErrorProcessor())
2108     if self.save_cookies:
2109       self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies_" + server)
2110       self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
2111       if os.path.exists(self.cookie_file):
2112         try:
2113           self.cookie_jar.load()
2114           self.authenticated = True
2115           StatusUpdate("Loaded authentication cookies from %s" %
2116                        self.cookie_file)
2117         except (cookielib.LoadError, IOError):
2118           # Failed to load cookies - just ignore them.
2119           pass
2120       else:
2121         # Create an empty cookie file with mode 600
2122         fd = os.open(self.cookie_file, os.O_CREAT, 0600)
2123         os.close(fd)
2124       # Always chmod the cookie file
2125       os.chmod(self.cookie_file, 0600)
2126     else:
2127       # Don't save cookies across runs of update.py.
2128       self.cookie_jar = cookielib.CookieJar()
2129     opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
2130     return opener
2131 
2132 
2133 parser = optparse.OptionParser(usage="%prog [options] [-- diff_options]")
2134 parser.add_option("-y", "--assume_yes", action="store_true",
2135                   dest="assume_yes", default=False,
2136                   help="Assume that the answer to yes/no questions is 'yes'.")
2137 # Logging
2138 group = parser.add_option_group("Logging options")
2139 group.add_option("-q", "--quiet", action="store_const", const=0,
2140                  dest="verbose", help="Print errors only.")
2141 group.add_option("-v", "--verbose", action="store_const", const=2,
2142                  dest="verbose", default=1,
2143                  help="Print info level logs (default).")
2144 group.add_option("--noisy", action="store_const", const=3,
2145                  dest="verbose", help="Print all logs.")
2146 # Review server
2147 group = parser.add_option_group("Review server options")
2148 group.add_option("-s", "--server", action="store", dest="server",
2149                  default="codereview.appspot.com",
2150                  metavar="SERVER",
2151                  help=("The server to upload to. The format is host[:port]. "
2152                        "Defaults to '%default'."))
2153 group.add_option("-e", "--email", action="store", dest="email",
2154                  metavar="EMAIL", default=None,
2155                  help="The username to use. Will prompt if omitted.")
2156 group.add_option("-H", "--host", action="store", dest="host",
2157                  metavar="HOST", default=None,
2158                  help="Overrides the Host header sent with all RPCs.")
2159 group.add_option("--no_cookies", action="store_false",
2160                  dest="save_cookies", default=True,
2161                  help="Do not save authentication cookies to local disk.")
2162 # Issue
2163 group = parser.add_option_group("Issue options")
2164 group.add_option("-d", "--description", action="store", dest="description",
2165                  metavar="DESCRIPTION", default=None,
2166                  help="Optional description when creating an issue.")
2167 group.add_option("-f", "--description_file", action="store",
2168                  dest="description_file", metavar="DESCRIPTION_FILE",
2169                  default=None,
2170                  help="Optional path of a file that contains "
2171                       "the description when creating an issue.")
2172 group.add_option("-r", "--reviewers", action="store", dest="reviewers",
2173                  metavar="REVIEWERS", default=None,
2174                  help="Add reviewers (comma separated email addresses).")
2175 group.add_option("--cc", action="store", dest="cc",
2176                  metavar="CC", default=None,
2177                  help="Add CC (comma separated email addresses).")
2178 group.add_option("--private", action="store_true", dest="private",
2179                  default=False,
2180                  help="Make the issue restricted to reviewers and those CCed")
2181 # Upload options
2182 group = parser.add_option_group("Patch options")
2183 group.add_option("-m", "--message", action="store", dest="message",
2184                  metavar="MESSAGE", default=None,
2185                  help="A message to identify the patch. "
2186                       "Will prompt if omitted.")
2187 group.add_option("-i", "--issue", type="int", action="store",
2188                  metavar="ISSUE", default=None,
2189                  help="Issue number to which to add. Defaults to new issue.")
2190 group.add_option("--download_base", action="store_true",
2191                  dest="download_base", default=False,
2192                  help="Base files will be downloaded by the server "
2193                  "(side-by-side diffs may not work on files with CRs).")
2194 group.add_option("--rev", action="store", dest="revision",
2195                  metavar="REV", default=None,
2196                  help="Branch/tree/revision to diff against (used by DVCS).")
2197 group.add_option("--send_mail", action="store_true",
2198                  dest="send_mail", default=False,
2199                  help="Send notification email to reviewers.")
2200 group.add_option("--vcs", action="store", dest="vcs",
2201                  metavar="VCS", default=None,
2202                  help=("Version control system (optional, usually upload.py "
2203                        "already guesses the right VCS)."))
2204 
2205 
2206 def GetRpcServer(options):
2207   """Returns an instance of an AbstractRpcServer.
2208 
2209   Returns:
2210     A new AbstractRpcServer, on which RPC calls can be made.
2211   """
2212 
2213   rpc_server_class = HttpRpcServer
2214 
2215   def GetUserCredentials():
2216     """Prompts the user for a username and password."""
2217     email = options.email
2218     if email is None:
2219       email = GetEmail("Email (login for uploading to %s)" % options.server)
2220     password = getpass.getpass("Password for %s: " % email)
2221     return (email, password)
2222 
2223   # If this is the dev_appserver, use fake authentication.
2224   host = (options.host or options.server).lower()
2225   if host == "localhost" or host.startswith("localhost:"):
2226     email = options.email
2227     if email is None:
2228       email = "test@example.com"
2229       logging.info("Using debug user %s.  Override with --email" % email)
2230     server = rpc_server_class(
2231         options.server,
2232         lambda: (email, "password"),
2233         host_override=options.host,
2234         extra_headers={"Cookie":
2235                        'dev_appserver_login="%s:False"' % email},
2236         save_cookies=options.save_cookies)
2237     # Don't try to talk to ClientLogin.
2238     server.authenticated = True
2239     return server
2240 
2241   return rpc_server_class(options.server, GetUserCredentials,
2242                           host_override=options.host,
2243                           save_cookies=options.save_cookies)
2244 
2245 
2246 def EncodeMultipartFormData(fields, files):
2247   """Encode form fields for multipart/form-data.
2248 
2249   Args:
2250     fields: A sequence of (name, value) elements for regular form fields.
2251     files: A sequence of (name, filename, value) elements for data to be
2252            uploaded as files.
2253   Returns:
2254     (content_type, body) ready for httplib.HTTP instance.
2255 
2256   Source:
2257     http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
2258   """
2259   BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
2260   CRLF = '\r\n'
2261   lines = []
2262   for (key, value) in fields:
2263     lines.append('--' + BOUNDARY)
2264     lines.append('Content-Disposition: form-data; name="%s"' % key)
2265     lines.append('')
2266     if type(value) == unicode:
2267       value = value.encode("utf-8")
2268     lines.append(value)
2269   for (key, filename, value) in files:
2270     if type(filename) == unicode:
2271       filename = filename.encode("utf-8")
2272     if type(value) == unicode:
2273       value = value.encode("utf-8")
2274     lines.append('--' + BOUNDARY)
2275     lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
2276              (key, filename))
2277     lines.append('Content-Type: %s' % GetContentType(filename))
2278     lines.append('')
2279     lines.append(value)
2280   lines.append('--' + BOUNDARY + '--')
2281   lines.append('')
2282   body = CRLF.join(lines)
2283   content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
2284   return content_type, body
2285 
2286 
2287 def GetContentType(filename):
2288   """Helper to guess the content-type from the filename."""
2289   return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
2290 
2291 
2292 # Use a shell for subcommands on Windows to get a PATH search.
2293 use_shell = sys.platform.startswith("win")
2294 
2295 def RunShellWithReturnCode(command, print_output=False,
2296                            universal_newlines=True,
2297                            env=os.environ):
2298   """Executes a command and returns the output from stdout and the return code.
2299 
2300   Args:
2301     command: Command to execute.
2302     print_output: If True, the output is printed to stdout.
2303                   If False, both stdout and stderr are ignored.
2304     universal_newlines: Use universal_newlines flag (default: True).
2305 
2306   Returns:
2307     Tuple (output, return code)
2308   """
2309   logging.info("Running %s", command)
2310   p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2311                        shell=use_shell, universal_newlines=universal_newlines,
2312                        env=env)
2313   if print_output:
2314     output_array = []
2315     while True:
2316       line = p.stdout.readline()
2317       if not line:
2318         break
2319       print line.strip("\n")
2320       output_array.append(line)
2321     output = "".join(output_array)
2322   else:
2323     output = p.stdout.read()
2324   p.wait()
2325   errout = p.stderr.read()
2326   if print_output and errout:
2327     print >>sys.stderr, errout
2328   p.stdout.close()
2329   p.stderr.close()
2330   return output, p.returncode
2331 
2332 
2333 def RunShell(command, silent_ok=False, universal_newlines=True,
2334              print_output=False, env=os.environ):
2335   data, retcode = RunShellWithReturnCode(command, print_output,
2336                                          universal_newlines, env)
2337   if retcode:
2338     ErrorExit("Got error status from %s:\n%s" % (command, data))
2339   if not silent_ok and not data:
2340     ErrorExit("No output from %s" % command)
2341   return data
2342 
2343 
2344 class VersionControlSystem(object):
2345   """Abstract base class providing an interface to the VCS."""
2346 
2347   def __init__(self, options):
2348     """Constructor.
2349 
2350     Args:
2351       options: Command line options.
2352     """
2353     self.options = options
2354 
2355   def GenerateDiff(self, args):
2356     """Return the current diff as a string.
2357 
2358     Args:
2359       args: Extra arguments to pass to the diff command.
2360     """
2361     raise NotImplementedError(
2362         "abstract method -- subclass %s must override" % self.__class__)
2363 
2364   def GetUnknownFiles(self):
2365     """Return a list of files unknown to the VCS."""
2366     raise NotImplementedError(
2367         "abstract method -- subclass %s must override" % self.__class__)
2368 
2369   def CheckForUnknownFiles(self):
2370     """Show an "are you sure?" prompt if there are unknown files."""
2371     unknown_files = self.GetUnknownFiles()
2372     if unknown_files:
2373       print "The following files are not added to version control:"
2374       for line in unknown_files:
2375         print line
2376       prompt = "Are you sure to continue?(y/N) "
2377       answer = raw_input(prompt).strip()
2378       if answer != "y":
2379         ErrorExit("User aborted")
2380 
2381   def GetBaseFile(self, filename):
2382     """Get the content of the upstream version of a file.
2383 
2384     Returns:
2385       A tuple (base_content, new_content, is_binary, status)
2386         base_content: The contents of the base file.
2387         new_content: For text files, this is empty.  For binary files, this is
2388           the contents of the new file, since the diff output won't contain
2389           information to reconstruct the current file.
2390         is_binary: True iff the file is binary.
2391         status: The status of the file.
2392     """
2393 
2394     raise NotImplementedError(
2395         "abstract method -- subclass %s must override" % self.__class__)
2396 
2397 
2398   def GetBaseFiles(self, diff):
2399     """Helper that calls GetBase file for each file in the patch.
2400 
2401     Returns:
2402       A dictionary that maps from filename to GetBaseFile's tuple.  Filenames
2403       are retrieved based on lines that start with "Index:" or
2404       "Property changes on:".
2405     """
2406     files = {}
2407     for line in diff.splitlines(True):
2408       if line.startswith('Index:') or line.startswith('Property changes on:'):
2409         unused, filename = line.split(':', 1)
2410         # On Windows if a file has property changes its filename uses '\'
2411         # instead of '/'.
2412         filename = filename.strip().replace('\\', '/')
2413         files[filename] = self.GetBaseFile(filename)
2414     return files
2415 
2416 
2417   def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options,
2418                       files):
2419     """Uploads the base files (and if necessary, the current ones as well)."""
2420 
2421     def UploadFile(filename, file_id, content, is_binary, status, is_base):
2422       """Uploads a file to the server."""
2423       file_too_large = False
2424       if is_base:
2425         type = "base"
2426       else:
2427         type = "current"
2428       if len(content) > MAX_UPLOAD_SIZE:
2429         print ("Not uploading the %s file for %s because it's too large." %
2430                (type, filename))
2431         file_too_large = True
2432         content = ""
2433       checksum = md5(content).hexdigest()
2434       if options.verbose > 0 and not file_too_large:
2435         print "Uploading %s file for %s" % (type, filename)
2436       url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id)
2437       form_fields = [("filename", filename),
2438                      ("status", status),
2439                      ("checksum", checksum),
2440                      ("is_binary", str(is_binary)),
2441                      ("is_current", str(not is_base)),
2442                     ]
2443       if file_too_large:
2444         form_fields.append(("file_too_large", "1"))
2445       if options.email:
2446         form_fields.append(("user", options.email))
2447       ctype, body = EncodeMultipartFormData(form_fields,
2448                                             [("data", filename, content)])
2449       response_body = rpc_server.Send(url, body,
2450                                       content_type=ctype)
2451       if not response_body.startswith("OK"):
2452         StatusUpdate("  --> %s" % response_body)
2453         sys.exit(1)
2454 
2455     patches = dict()
2456     [patches.setdefault(v, k) for k, v in patch_list]
2457     for filename in patches.keys():
2458       base_content, new_content, is_binary, status = files[filename]
2459       file_id_str = patches.get(filename)
2460       if file_id_str.find("nobase") != -1:
2461         base_content = None
2462         file_id_str = file_id_str[file_id_str.rfind("_") + 1:]
2463       file_id = int(file_id_str)
2464       if base_content != None:
2465         UploadFile(filename, file_id, base_content, is_binary, status, True)
2466       if new_content != None:
2467         UploadFile(filename, file_id, new_content, is_binary, status, False)
2468 
2469   def IsImage(self, filename):
2470     """Returns true if the filename has an image extension."""
2471     mimetype =  mimetypes.guess_type(filename)[0]
2472     if not mimetype:
2473       return False
2474     return mimetype.startswith("image/")
2475 
2476   def IsBinary(self, filename):
2477     """Returns true if the guessed mimetyped isnt't in text group."""
2478     mimetype = mimetypes.guess_type(filename)[0]
2479     if not mimetype:
2480       return False  # e.g. README, "real" binaries usually have an extension
2481     # special case for text files which don't start with text/
2482     if mimetype in TEXT_MIMETYPES:
2483       return False
2484     return not mimetype.startswith("text/")
2485 
2486 
2487 class SubversionVCS(VersionControlSystem):
2488   """Implementation of the VersionControlSystem interface for Subversion."""
2489 
2490   def __init__(self, options):
2491     super(SubversionVCS, self).__init__(options)
2492     if self.options.revision:
2493       match = re.match(r"(\d+)(:(\d+))?", self.options.revision)
2494       if not match:
2495         ErrorExit("Invalid Subversion revision %s." % self.options.revision)
2496       self.rev_start = match.group(1)
2497       self.rev_end = match.group(3)
2498     else:
2499       self.rev_start = self.rev_end = None
2500     # Cache output from "svn list -r REVNO dirname".
2501     # Keys: dirname, Values: 2-tuple (ouput for start rev and end rev).
2502     self.svnls_cache = {}
2503     # SVN base URL is required to fetch files deleted in an older revision.
2504     # Result is cached to not guess it over and over again in GetBaseFile().
2505     required = self.options.download_base or self.options.revision is not None
2506     self.svn_base = self._GuessBase(required)
2507 
2508   def GuessBase(self, required):
2509     """Wrapper for _GuessBase."""
2510     return self.svn_base
2511 
2512   def _GuessBase(self, required):
2513     """Returns the SVN base URL.
2514 
2515     Args:
2516       required: If true, exits if the url can't be guessed, otherwise None is
2517         returned.
2518     """
2519     info = RunShell(["svn", "info"])
2520     for line in info.splitlines():
2521       words = line.split()
2522       if len(words) == 2 and words[0] == "URL:":
2523         url = words[1]
2524         scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
2525         username, netloc = urllib.splituser(netloc)
2526         if username:
2527           logging.info("Removed username from base URL")
2528         if netloc.endswith("svn.python.org"):
2529           if netloc == "svn.python.org":
2530             if path.startswith("/projects/"):
2531               path = path[9:]
2532           elif netloc != "pythondev@svn.python.org":
2533             ErrorExit("Unrecognized Python URL: %s" % url)
2534           base = "http://svn.python.org/view/*checkout*%s/" % path
2535           logging.info("Guessed Python base = %s", base)
2536         elif netloc.endswith("svn.collab.net"):
2537           if path.startswith("/repos/"):
2538             path = path[6:]
2539           base = "http://svn.collab.net/viewvc/*checkout*%s/" % path
2540           logging.info("Guessed CollabNet base = %s", base)
2541         elif netloc.endswith(".googlecode.com"):
2542           path = path + "/"
2543           base = urlparse.urlunparse(("http", netloc, path, params,
2544                                       query, fragment))
2545           logging.info("Guessed Google Code base = %s", base)
2546         else:
2547           path = path + "/"
2548           base = urlparse.urlunparse((scheme, netloc, path, params,
2549                                       query, fragment))
2550           logging.info("Guessed base = %s", base)
2551         return base
2552     if required:
2553       ErrorExit("Can't find URL in output from svn info")
2554     return None
2555 
2556   def GenerateDiff(self, args):
2557     cmd = ["svn", "diff"]
2558     if self.options.revision:
2559       cmd += ["-r", self.options.revision]
2560     cmd.extend(args)
2561     data = RunShell(cmd)
2562     count = 0
2563     for line in data.splitlines():
2564       if line.startswith("Index:") or line.startswith("Property changes on:"):
2565         count += 1
2566         logging.info(line)
2567     if not count:
2568       ErrorExit("No valid patches found in output from svn diff")
2569     return data
2570 
2571   def _CollapseKeywords(self, content, keyword_str):
2572     """Collapses SVN keywords."""
2573     # svn cat translates keywords but svn diff doesn't. As a result of this
2574     # behavior patching.PatchChunks() fails with a chunk mismatch error.
2575     # This part was originally written by the Review Board development team
2576     # who had the same problem (http://reviews.review-board.org/r/276/).
2577     # Mapping of keywords to known aliases
2578     svn_keywords = {
2579       # Standard keywords
2580       'Date':                ['Date', 'LastChangedDate'],
2581       'Revision':            ['Revision', 'LastChangedRevision', 'Rev'],
2582       'Author':              ['Author', 'LastChangedBy'],
2583       'HeadURL':             ['HeadURL', 'URL'],
2584       'Id':                  ['Id'],
2585 
2586       # Aliases
2587       'LastChangedDate':     ['LastChangedDate', 'Date'],
2588       'LastChangedRevision': ['LastChangedRevision', 'Rev', 'Revision'],
2589       'LastChangedBy':       ['LastChangedBy', 'Author'],
2590       'URL':                 ['URL', 'HeadURL'],
2591     }
2592 
2593     def repl(m):
2594        if m.group(2):
2595          return "$%s::%s$" % (m.group(1), " " * len(m.group(3)))
2596        return "$%s$" % m.group(1)
2597     keywords = [keyword
2598                 for name in keyword_str.split(" ")
2599                 for keyword in svn_keywords.get(name, [])]
2600     return re.sub(r"\$(%s):(:?)([^\$]+)\$" % '|'.join(keywords), repl, content)
2601 
2602   def GetUnknownFiles(self):
2603     status = RunShell(["svn", "status", "--ignore-externals"], silent_ok=True)
2604     unknown_files = []
2605     for line in status.split("\n"):
2606       if line and line[0] == "?":
2607         unknown_files.append(line)
2608     return unknown_files
2609 
2610   def ReadFile(self, filename):
2611     """Returns the contents of a file."""
2612     file = open(filename, 'rb')
2613     result = ""
2614     try:
2615       result = file.read()
2616     finally:
2617       file.close()
2618     return result
2619 
2620   def GetStatus(self, filename):
2621     """Returns the status of a file."""
2622     if not self.options.revision:
2623       status = RunShell(["svn", "status", "--ignore-externals", filename])
2624       if not status:
2625         ErrorExit("svn status returned no output for %s" % filename)
2626       status_lines = status.splitlines()
2627       # If file is in a cl, the output will begin with
2628       # "\n--- Changelist 'cl_name':\n".  See
2629       # http://svn.collab.net/repos/svn/trunk/notes/changelist-design.txt
2630       if (len(status_lines) == 3 and
2631           not status_lines[0] and
2632           status_lines[1].startswith("--- Changelist")):
2633         status = status_lines[2]
2634       else:
2635         status = status_lines[0]
2636     # If we have a revision to diff against we need to run "svn list"
2637     # for the old and the new revision and compare the results to get
2638     # the correct status for a file.
2639     else:
2640       dirname, relfilename = os.path.split(filename)
2641       if dirname not in self.svnls_cache:
2642         cmd = ["svn", "list", "-r", self.rev_start, dirname or "."]
2643         out, returncode = RunShellWithReturnCode(cmd)
2644         if returncode:
2645           ErrorExit("Failed to get status for %s." % filename)
2646         old_files = out.splitlines()
2647         args = ["svn", "list"]
2648         if self.rev_end:
2649           args += ["-r", self.rev_end]
2650         cmd = args + [dirname or "."]
2651         out, returncode = RunShellWithReturnCode(cmd)
2652         if returncode:
2653           ErrorExit("Failed to run command %s" % cmd)
2654         self.svnls_cache[dirname] = (old_files, out.splitlines())
2655       old_files, new_files = self.svnls_cache[dirname]
2656       if relfilename in old_files and relfilename not in new_files:
2657         status = "D   "
2658       elif relfilename in old_files and relfilename in new_files:
2659         status = "M   "
2660       else:
2661         status = "A   "
2662     return status
2663 
2664   def GetBaseFile(self, filename):
2665     status = self.GetStatus(filename)
2666     base_content = None
2667     new_content = None
2668 
2669     # If a file is copied its status will be "A  +", which signifies
2670     # "addition-with-history".  See "svn st" for more information.  We need to
2671     # upload the original file or else diff parsing will fail if the file was
2672     # edited.
2673     if status[0] == "A" and status[3] != "+":
2674       # We'll need to upload the new content if we're adding a binary file
2675       # since diff's output won't contain it.
2676       mimetype = RunShell(["svn", "propget", "svn:mime-type", filename],
2677                           silent_ok=True)
2678       base_content = ""
2679       is_binary = bool(mimetype) and not mimetype.startswith("text/")
2680       if is_binary and self.IsImage(filename):
2681         new_content = self.ReadFile(filename)
2682     elif (status[0] in ("M", "D", "R") or
2683           (status[0] == "A" and status[3] == "+") or  # Copied file.
2684           (status[0] == " " and status[1] == "M")):  # Property change.
2685       args = []
2686       if self.options.revision:
2687         url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
2688       else:
2689         # Don't change filename, it's needed later.
2690         url = filename
2691         args += ["-r", "BASE"]
2692       cmd = ["svn"] + args + ["propget", "svn:mime-type", url]
2693       mimetype, returncode = RunShellWithReturnCode(cmd)
2694       if returncode:
2695         # File does not exist in the requested revision.
2696         # Reset mimetype, it contains an error message.
2697         mimetype = ""
2698       get_base = False
2699       is_binary = bool(mimetype) and not mimetype.startswith("text/")
2700       if status[0] == " ":
2701         # Empty base content just to force an upload.
2702         base_content = ""
2703       elif is_binary:
2704         if self.IsImage(filename):
2705           get_base = True
2706           if status[0] == "M":
2707             if not self.rev_end:
2708               new_content = self.ReadFile(filename)
2709             else:
2710               url = "%s/%s@%s" % (self.svn_base, filename, self.rev_end)
2711               new_content = RunShell(["svn", "cat", url],
2712                                      universal_newlines=True, silent_ok=True)
2713         else:
2714           base_content = ""
2715       else:
2716         get_base = True
2717 
2718       if get_base:
2719         if is_binary:
2720           universal_newlines = False
2721         else:
2722           universal_newlines = True
2723         if self.rev_start:
2724           # "svn cat -r REV delete_file.txt" doesn't work. cat requires
2725           # the full URL with "@REV" appended instead of using "-r" option.
2726           url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
2727           base_content = RunShell(["svn", "cat", url],
2728                                   universal_newlines=universal_newlines,
2729                                   silent_ok=True)
2730         else:
2731           base_content = RunShell(["svn", "cat", filename],
2732                                   universal_newlines=universal_newlines,
2733                                   silent_ok=True)
2734         if not is_binary:
2735           args = []
2736           if self.rev_start:
2737             url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
2738           else:
2739             url = filename
2740             args += ["-r", "BASE"]
2741           cmd = ["svn"] + args + ["propget", "svn:keywords", url]
2742           keywords, returncode = RunShellWithReturnCode(cmd)
2743           if keywords and not returncode:
2744             base_content = self._CollapseKeywords(base_content, keywords)
2745     else:
2746       StatusUpdate("svn status returned unexpected output: %s" % status)
2747       sys.exit(1)
2748     return base_content, new_content, is_binary, status[0:5]
2749 
2750 
2751 class GitVCS(VersionControlSystem):
2752   """Implementation of the VersionControlSystem interface for Git."""
2753 
2754   def __init__(self, options):
2755     super(GitVCS, self).__init__(options)
2756     # Map of filename -> (hash before, hash after) of base file.
2757     # Hashes for "no such file" are represented as None.
2758     self.hashes = {}
2759     # Map of new filename -> old filename for renames.
2760     self.renames = {}
2761 
2762   def GenerateDiff(self, extra_args):
2763     # This is more complicated than svn's GenerateDiff because we must convert
2764     # the diff output to include an svn-style "Index:" line as well as record
2765     # the hashes of the files, so we can upload them along with our diff.
2766 
2767     # Special used by git to indicate "no such content".
2768     NULL_HASH = "0"*40
2769 
2770     extra_args = extra_args[:]
2771     if self.options.revision:
2772       extra_args = [self.options.revision] + extra_args
2773     extra_args.append('-M')
2774 
2775     # --no-ext-diff is broken in some versions of Git, so try to work around
2776     # this by overriding the environment (but there is still a problem if the
2777     # git config key "diff.external" is used).
2778     env = os.environ.copy()
2779     if 'GIT_EXTERNAL_DIFF' in env: del env['GIT_EXTERNAL_DIFF']
2780     gitdiff = RunShell(["git", "diff", "--no-ext-diff", "--full-index"]
2781                        + extra_args, env=env)
2782     svndiff = []
2783     filecount = 0
2784     filename = None
2785     for line in gitdiff.splitlines():
2786       match = re.match(r"diff --git a/(.*) b/(.*)$", line)
2787       if match:
2788         filecount += 1
2789         # Intentionally use the "after" filename so we can show renames.
2790         filename = match.group(2)
2791         svndiff.append("Index: %s\n" % filename)
2792         if match.group(1) != match.group(2):
2793           self.renames[match.group(2)] = match.group(1)
2794       else:
2795         # The "index" line in a git diff looks like this (long hashes elided):
2796         #   index 82c0d44..b2cee3f 100755
2797         # We want to save the left hash, as that identifies the base file.
2798         match = re.match(r"index (\w+)\.\.(\w+)", line)
2799         if match:
2800           before, after = (match.group(1), match.group(2))
2801           if before == NULL_HASH:
2802             before = None
2803           if after == NULL_HASH:
2804             after = None
2805           self.hashes[filename] = (before, after)
2806       svndiff.append(line + "\n")
2807     if not filecount:
2808       ErrorExit("No valid patches found in output from git diff")
2809     return "".join(svndiff)
2810 
2811   def GetUnknownFiles(self):
2812     status = RunShell(["git", "ls-files", "--exclude-standard", "--others"],
2813                       silent_ok=True)
2814     return status.splitlines()
2815 
2816   def GetFileContent(self, file_hash, is_binary):
2817     """Returns the content of a file identified by its git hash."""
2818     data, retcode = RunShellWithReturnCode(["git", "show", file_hash],
2819                                             universal_newlines=not is_binary)
2820     if retcode:
2821       ErrorExit("Got error status from 'git show %s'" % file_hash)
2822     return data
2823 
2824   def GetBaseFile(self, filename):
2825     hash_before, hash_after = self.hashes.get(filename, (None,None))
2826     base_content = None
2827     new_content = None
2828     is_binary = self.IsBinary(filename)
2829     status = None
2830 
2831     if filename in self.renames:
2832       status = "A +"  # Match svn attribute name for renames.
2833       if filename not in self.hashes:
2834         # If a rename doesn't change the content, we never get a hash.
2835         base_content = RunShell(["git", "show", filename])
2836     elif not hash_before:
2837       status = "A"
2838       base_content = ""
2839     elif not hash_after:
2840       status = "D"
2841     else:
2842       status = "M"
2843 
2844     is_image = self.IsImage(filename)
2845 
2846     # Grab the before/after content if we need it.
2847     # We should include file contents if it's text or it's an image.
2848     if not is_binary or is_image:
2849       # Grab the base content if we don't have it already.
2850       if base_content is None and hash_before:
2851         base_content = self.GetFileContent(hash_before, is_binary)
2852       # Only include the "after" file if it's an image; otherwise it
2853       # it is reconstructed from the diff.
2854       if is_image and hash_after:
2855         new_content = self.GetFileContent(hash_after, is_binary)
2856 
2857     return (base_content, new_content, is_binary, status)
2858 
2859 
2860 class MercurialVCS(VersionControlSystem):
2861   """Implementation of the VersionControlSystem interface for Mercurial."""
2862 
2863   def __init__(self, options, repo_dir):
2864     super(MercurialVCS, self).__init__(options)
2865     # Absolute path to repository (we can be in a subdir)
2866     self.repo_dir = os.path.normpath(repo_dir)
2867     # Compute the subdir
2868     cwd = os.path.normpath(os.getcwd())
2869     assert cwd.startswith(self.repo_dir)
2870     self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/")
2871     if self.options.revision:
2872       self.base_rev = self.options.revision
2873     else:
2874       mqparent, err = RunShellWithReturnCode(['hg', 'log', '--rev', 'qparent', '--template={node}'])
2875       if not err:
2876         self.base_rev = mqparent
2877       else:
2878         self.base_rev = RunShell(["hg", "parents", "-q"]).split(':')[1].strip()
2879   def _GetRelPath(self, filename):
2880     """Get relative path of a file according to the current directory,
2881     given its logical path in the repo."""
2882     assert filename.startswith(self.subdir), (filename, self.subdir)
2883     return filename[len(self.subdir):].lstrip(r"\/")
2884 
2885   def GenerateDiff(self, extra_args):
2886     # If no file specified, restrict to the current subdir
2887     extra_args = extra_args or ["."]
2888     cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
2889     data = RunShell(cmd, silent_ok=True)
2890     svndiff = []
2891     filecount = 0
2892     for line in data.splitlines():
2893       m = re.match("diff --git a/(\S+) b/(\S+)", line)
2894       if m:
2895         # Modify line to make it look like as it comes from svn diff.
2896         # With this modification no changes on the server side are required
2897         # to make upload.py work with Mercurial repos.
2898         # NOTE: for proper handling of moved/copied files, we have to use
2899         # the second filename.
2900         filename = m.group(2)
2901         svndiff.append("Index: %s" % filename)
2902         svndiff.append("=" * 67)
2903         filecount += 1
2904         logging.info(line)
2905       else:
2906         svndiff.append(line)
2907     if not filecount:
2908       ErrorExit("No valid patches found in output from hg diff")
2909     return "\n".join(svndiff) + "\n"
2910 
2911   def GetUnknownFiles(self):
2912     """Return a list of files unknown to the VCS."""
2913     args = []
2914     status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
2915         silent_ok=True)
2916     unknown_files = []
2917     for line in status.splitlines():
2918       st, fn = line.split(" ", 1)
2919       if st == "?":
2920         unknown_files.append(fn)
2921     return unknown_files
2922 
2923   def GetBaseFile(self, filename):
2924     # "hg status" and "hg cat" both take a path relative to the current subdir
2925     # rather than to the repo root, but "hg diff" has given us the full path
2926     # to the repo root.
2927     base_content = ""
2928     new_content = None
2929     is_binary = False
2930     oldrelpath = relpath = self._GetRelPath(filename)
2931     # "hg status -C" returns two lines for moved/copied files, one otherwise
2932     out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath])
2933     out = out.splitlines()
2934     # HACK: strip error message about missing file/directory if it isn't in
2935     # the working copy
2936     if out[0].startswith('%s: ' % relpath):
2937       out = out[1:]
2938     status, what = out[0].split(' ', 1)
2939     if len(out) > 1 and status == "A" and what == relpath:
2940       oldrelpath = out[1].strip()
2941       status = "M"
2942     if ":" in self.base_rev:
2943       base_rev = self.base_rev.split(":", 1)[0]
2944     else:
2945       base_rev = self.base_rev
2946     if status != "A":
2947       base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
2948         silent_ok=True)
2949       is_binary = "\0" in base_content  # Mercurial's heuristic
2950     if status != "R":
2951       new_content = open(relpath, "rb").read()
2952       is_binary = is_binary or "\0" in new_content
2953     if is_binary and base_content:
2954       # Fetch again without converting newlines
2955       base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
2956         silent_ok=True, universal_newlines=False)
2957     if not is_binary or not self.IsImage(relpath):
2958       new_content = None
2959     return base_content, new_content, is_binary, status
2960 
2961 
2962 # NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
2963 def SplitPatch(data):
2964   """Splits a patch into separate pieces for each file.
2965 
2966   Args:
2967     data: A string containing the output of svn diff.
2968 
2969   Returns:
2970     A list of 2-tuple (filename, text) where text is the svn diff output
2971       pertaining to filename.
2972   """
2973   patches = []
2974   filename = None
2975   diff = []
2976   for line in data.splitlines(True):
2977     new_filename = None
2978     if line.startswith('Index:'):
2979       unused, new_filename = line.split(':', 1)
2980       new_filename = new_filename.strip()
2981     elif line.startswith('Property changes on:'):
2982       unused, temp_filename = line.split(':', 1)
2983       # When a file is modified, paths use '/' between directories, however
2984       # when a property is modified '\' is used on Windows.  Make them the same
2985       # otherwise the file shows up twice.
2986       temp_filename = temp_filename.strip().replace('\\', '/')
2987       if temp_filename != filename:
2988         # File has property changes but no modifications, create a new diff.
2989         new_filename = temp_filename
2990     if new_filename:
2991       if filename and diff:
2992         patches.append((filename, ''.join(diff)))
2993       filename = new_filename
2994       diff = [line]
2995       continue
2996     if diff is not None:
2997       diff.append(line)
2998   if filename and diff:
2999     patches.append((filename, ''.join(diff)))
3000   return patches
3001 
3002 
3003 def UploadSeparatePatches(issue, rpc_server, patchset, data, options):
3004   """Uploads a separate patch for each file in the diff output.
3005 
3006   Returns a list of [patch_key, filename] for each file.
3007   """
3008   patches = SplitPatch(data)
3009   rv = []
3010   for patch in patches:
3011     if len(patch[1]) > MAX_UPLOAD_SIZE:
3012       print ("Not uploading the patch for " + patch[0] +
3013              " because the file is too large.")
3014       continue
3015     form_fields = [("filename", patch[0])]
3016     if not options.download_base:
3017       form_fields.append(("content_upload", "1"))
3018     files = [("data", "data.diff", patch[1])]
3019     ctype, body = EncodeMultipartFormData(form_fields, files)
3020     url = "/%d/upload_patch/%d" % (int(issue), int(patchset))
3021     print "Uploading patch for " + patch[0]
3022     response_body = rpc_server.Send(url, body, content_type=ctype)
3023     lines = response_body.splitlines()
3024     if not lines or lines[0] != "OK":
3025       StatusUpdate("  --> %s" % response_body)
3026       sys.exit(1)
3027     rv.append([lines[1], patch[0]])
3028   return rv
3029 
3030 
3031 def GuessVCSName():
3032   """Helper to guess the version control system.
3033 
3034   This examines the current directory, guesses which VersionControlSystem
3035   we're using, and returns an string indicating which VCS is detected.
3036 
3037   Returns:
3038     A pair (vcs, output).  vcs is a string indicating which VCS was detected
3039     and is one of VCS_GIT, VCS_MERCURIAL, VCS_SUBVERSION, or VCS_UNKNOWN.
3040     output is a string containing any interesting output from the vcs
3041     detection routine, or None if there is nothing interesting.
3042   """
3043   # Mercurial has a command to get the base directory of a repository
3044   # Try running it, but don't die if we don't have hg installed.
3045   # NOTE: we try Mercurial first as it can sit on top of an SVN working copy.
3046   try:
3047     out, returncode = RunShellWithReturnCode(["hg", "root"])
3048     if returncode == 0:
3049       return (VCS_MERCURIAL, out.strip())
3050   except OSError, (errno, message):
3051     if errno != 2:  # ENOENT -- they don't have hg installed.
3052       raise
3053 
3054   # Subversion has a .svn in all working directories.
3055   if os.path.isdir('.svn'):
3056     logging.info("Guessed VCS = Subversion")
3057     return (VCS_SUBVERSION, None)
3058 
3059   # Git has a command to test if you're in a git tree.
3060   # Try running it, but don't die if we don't have git installed.
3061   try:
3062     out, returncode = RunShellWithReturnCode(["git", "rev-parse",
3063                                               "--is-inside-work-tree"])
3064     if returncode == 0:
3065       return (VCS_GIT, None)
3066   except OSError, (errno, message):
3067     if errno != 2:  # ENOENT -- they don't have git installed.
3068       raise
3069 
3070   return (VCS_UNKNOWN, None)
3071 
3072 
3073 def GuessVCS(options):
3074   """Helper to guess the version control system.
3075 
3076   This verifies any user-specified VersionControlSystem (by command line
3077   or environment variable).  If the user didn't specify one, this examines
3078   the current directory, guesses which VersionControlSystem we're using,
3079   and returns an instance of the appropriate class.  Exit with an error
3080   if we can't figure it out.
3081 
3082   Returns:
3083     A VersionControlSystem instance. Exits if the VCS can't be guessed.
3084   """
3085   vcs = options.vcs
3086   if not vcs:
3087     vcs = os.environ.get("CODEREVIEW_VCS")
3088   if vcs:
3089     v = VCS_ABBREVIATIONS.get(vcs.lower())
3090     if v is None:
3091       ErrorExit("Unknown version control system %r specified." % vcs)
3092     (vcs, extra_output) = (v, None)
3093   else:
3094     (vcs, extra_output) = GuessVCSName()
3095 
3096   if vcs == VCS_MERCURIAL:
3097     if extra_output is None:
3098       extra_output = RunShell(["hg", "root"]).strip()
3099     return MercurialVCS(options, extra_output)
3100   elif vcs == VCS_SUBVERSION:
3101     return SubversionVCS(options)
3102   elif vcs == VCS_GIT:
3103     return GitVCS(options)
3104 
3105   ErrorExit(("Could not guess version control system. "
3106              "Are you in a working copy directory?"))
3107 
3108 
3109 def RealMain(argv, data=None):
3110   """The real main function.
3111 
3112   Args:
3113     argv: Command line arguments.
3114     data: Diff contents. If None (default) the diff is generated by
3115       the VersionControlSystem implementation returned by GuessVCS().
3116 
3117   Returns:
3118     A 2-tuple (issue id, patchset id).
3119     The patchset id is None if the base files are not uploaded by this
3120     script (applies only to SVN checkouts).
3121   """
3122   logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:"
3123                               "%(lineno)s %(message)s "))
3124   os.environ['LC_ALL'] = 'C'
3125   options, args = parser.parse_args(argv[1:])
3126   global verbosity
3127   verbosity = options.verbose
3128   if verbosity >= 3:
3129     logging.getLogger().setLevel(logging.DEBUG)
3130   elif verbosity >= 2:
3131     logging.getLogger().setLevel(logging.INFO)
3132   vcs = GuessVCS(options)
3133   if isinstance(vcs, SubversionVCS):
3134     # base field is only allowed for Subversion.
3135     # Note: Fetching base files may become deprecated in future releases.
3136     base = vcs.GuessBase(options.download_base)
3137   else:
3138     base = None
3139   if not base and options.download_base:
3140     options.download_base = True
3141     logging.info("Enabled upload of base file")
3142   if not options.assume_yes:
3143     vcs.CheckForUnknownFiles()
3144   if data is None:
3145     data = vcs.GenerateDiff(args)
3146   files = vcs.GetBaseFiles(data)
3147   if verbosity >= 1:
3148     print "Upload server:", options.server, "(change with -s/--server)"
3149   if options.issue:
3150     prompt = "Message describing this patch set: "
3151   else:
3152     prompt = "New issue subject: "
3153   message = options.message or raw_input(prompt).strip()
3154   if not message:
3155     ErrorExit("A non-empty message is required")
3156   rpc_server = GetRpcServer(options)
3157   form_fields = [("subject", message)]
3158   if base:
3159     form_fields.append(("base", base))
3160   if options.issue:
3161     form_fields.append(("issue", str(options.issue)))
3162   if options.email:
3163     form_fields.append(("user", options.email))
3164   if options.reviewers:
3165     for reviewer in options.reviewers.split(','):
3166       if "@" in reviewer and not reviewer.split("@")[1].count(".") == 1:
3167         ErrorExit("Invalid email address: %s" % reviewer)
3168     form_fields.append(("reviewers", options.reviewers))
3169   if options.cc:
3170     for cc in options.cc.split(','):
3171       if "@" in cc and not cc.split("@")[1].count(".") == 1:
3172         ErrorExit("Invalid email address: %s" % cc)
3173     form_fields.append(("cc", options.cc))
3174   description = options.description
3175   if options.description_file:
3176     if options.description:
3177       ErrorExit("Can't specify description and description_file")
3178     file = open(options.description_file, 'r')
3179     description = file.read()
3180     file.close()
3181   if description:
3182     form_fields.append(("description", description))
3183   # Send a hash of all the base file so the server can determine if a copy
3184   # already exists in an earlier patchset.
3185   base_hashes = ""
3186   for file, info in files.iteritems():
3187     if not info[0] is None:
3188       checksum = md5(info[0]).hexdigest()
3189       if base_hashes:
3190         base_hashes += "|"
3191       base_hashes += checksum + ":" + file
3192   form_fields.append(("base_hashes", base_hashes))
3193   if options.private:
3194     if options.issue:
3195       print "Warning: Private flag ignored when updating an existing issue."
3196     else:
3197       form_fields.append(("private", "1"))
3198   # If we're uploading base files, don't send the email before the uploads, so
3199   # that it contains the file status.
3200   if options.send_mail and options.download_base:
3201     form_fields.append(("send_mail", "1"))
3202   if not options.download_base:
3203     form_fields.append(("content_upload", "1"))
3204   if len(data) > MAX_UPLOAD_SIZE:
3205     print "Patch is large, so uploading file patches separately."
3206     uploaded_diff_file = []
3207     form_fields.append(("separate_patches", "1"))
3208   else:
3209     uploaded_diff_file = [("data", "data.diff", data)]
3210   ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
3211   response_body = rpc_server.Send("/upload", body, content_type=ctype)
3212   patchset = None
3213   if not options.download_base or not uploaded_diff_file:
3214     lines = response_body.splitlines()
3215     if len(lines) >= 2:
3216       msg = lines[0]
3217       patchset = lines[1].strip()
3218       patches = [x.split(" ", 1) for x in lines[2:]]
3219     else:
3220       msg = response_body
3221   else:
3222     msg = response_body
3223   if not response_body.startswith("Issue created.") and \
3224   not response_body.startswith("Issue updated."):
3225     print >>sys.stderr, msg
3226     sys.exit(0)
3227   issue = msg[msg.rfind("/")+1:]
3228 
3229   if not uploaded_diff_file:
3230     result = UploadSeparatePatches(issue, rpc_server, patchset, data, options)
3231     if not options.download_base:
3232       patches = result
3233 
3234   if not options.download_base:
3235     vcs.UploadBaseFiles(issue, rpc_server, patches, patchset, options, files)
3236     if options.send_mail:
3237       rpc_server.Send("/" + issue + "/mail", payload="")
3238   return issue, patchset
3239 
3240 
3241 def main():
3242   try:
3243     RealMain(sys.argv)
3244   except KeyboardInterrupt:
3245     print
3246     StatusUpdate("Interrupted.")
3247     sys.exit(1)
3248