1 #!/usr/bin/env python
2 #
3 # Copyright 2007-2009 Google Inc.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 #     http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 
17 '''Mercurial interface to codereview.appspot.com.
18 
19 To configure, set the following options in
20 your repository's .hg/hgrc file.
21 
22 	[extensions]
23 	codereview = path/to/codereview.py
24 
25 	[codereview]
26 	server = codereview.appspot.com
27 
28 The server should be running Rietveld; see http://code.google.com/p/rietveld/.
29 
30 In addition to the new commands, this extension introduces
31 the file pattern syntax @nnnnnn, where nnnnnn is a change list
32 number, to mean the files included in that change list, which
33 must be associated with the current client.
34 
35 For example, if change 123456 contains the files x.go and y.go,
36 "hg diff @123456" is equivalent to"hg diff x.go y.go".
37 '''
38 
39 from mercurial import cmdutil, commands, hg, util, error, match
40 from mercurial.node import nullrev, hex, nullid, short
41 import os, re, time
42 import stat
43 import subprocess
44 import threading
45 from HTMLParser import HTMLParser
46 try:
47 	from xml.etree import ElementTree as ET
48 except:
49 	from elementtree import ElementTree as ET
50 
51 try:
52 	hgversion = util.version()
53 except:
54 	from mercurial.version import version as v
55 	hgversion = v.get_version()
56 
57 oldMessage = """
58 The code review extension requires Mercurial 1.3 or newer.
59 
60 To install a new Mercurial,
61 
62 	sudo easy_install mercurial
63 
64 works on most systems.
65 """
66 
67 linuxMessage = """
68 You may need to clear your current Mercurial installation by running:
69 
70 	sudo apt-get remove mercurial mercurial-common
71 	sudo rm -rf /etc/mercurial
72 """
73 
74 if hgversion < '1.3':
75 	msg = oldMessage
76 	if os.access("/etc/mercurial", 0):
77 		msg += linuxMessage
78 	raise util.Abort(msg)
79 
80 # To experiment with Mercurial in the python interpreter:
81 #    >>> repo = hg.repository(ui.ui(), path = ".")
82 
83 #######################################################################
84 # Normally I would split this into multiple files, but it simplifies
85 # import path headaches to keep it all in one file.  Sorry.
86 
87 import sys
88 if __name__ == "__main__":
89 	print >>sys.stderr, "This is a Mercurial extension and should not be invoked directly."
90 	sys.exit(2)
91 
92 server = "codereview.appspot.com"
93 server_url_base = None
94 defaultcc = None
95 
96 #######################################################################
97 # Change list parsing.
98 #
99 # Change lists are stored in .hg/codereview/cl.nnnnnn
100 # where nnnnnn is the number assigned by the code review server.
101 # Most data about a change list is stored on the code review server
102 # too: the description, reviewer, and cc list are all stored there.
103 # The only thing in the cl.nnnnnn file is the list of relevant files.
104 # Also, the existence of the cl.nnnnnn file marks this repository
105 # as the one where the change list lives.
106 
107 emptydiff = """Index: ~rietveld~placeholder~
108 ===================================================================
109 diff --git a/~rietveld~placeholder~ b/~rietveld~placeholder~
110 new file mode 100644
111 """
112 
113 
114 class CL(object):
115 	def __init__(self, name):
116 		self.name = name
117 		self.desc = ''
118 		self.files = []
119 		self.reviewer = []
120 		self.cc = []
121 		self.url = ''
122 		self.local = False
123 		self.web = False
124 		self.copied_from = None	# None means current user
125 		self.mailed = False
126 
127 	def DiskText(self):
128 		cl = self
129 		s = ""
130 		if cl.copied_from:
131 			s += "Author: " + cl.copied_from + "\n\n"
132 		s += "Mailed: " + str(self.mailed) + "\n"
133 		s += "Description:\n"
134 		s += Indent(cl.desc, "\t")
135 		s += "Files:\n"
136 		for f in cl.files:
137 			s += "\t" + f + "\n"
138 		return s
139 
140 	def EditorText(self):
141 		cl = self
142 		s = _change_prolog
143 		s += "\n"
144 		if cl.copied_from:
145 			s += "Author: " + cl.copied_from + "\n"
146 		if cl.url != '':
147 			s += 'URL: ' + cl.url + '	# cannot edit\n\n'
148 		s += "Reviewer: " + JoinComma(cl.reviewer) + "\n"
149 		s += "CC: " + JoinComma(cl.cc) + "\n"
150 		s += "\n"
151 		s += "Description:\n"
152 		if cl.desc == '':
153 			s += "\t<enter description here>\n"
154 		else:
155 			s += Indent(cl.desc, "\t")
156 		s += "\n"
157 		if cl.local or cl.name == "new":
158 			s += "Files:\n"
159 			for f in cl.files:
160 				s += "\t" + f + "\n"
161 			s += "\n"
162 		return s
163 
164 	def PendingText(self):
165 		cl = self
166 		s = cl.name + ":" + "\n"
167 		s += Indent(cl.desc, "\t")
168 		s += "\n"
169 		if cl.copied_from:
170 			s += "\tAuthor: " + cl.copied_from + "\n"
171 		s += "\tReviewer: " + JoinComma(cl.reviewer) + "\n"
172 		s += "\tCC: " + JoinComma(cl.cc) + "\n"
173 		s += "\tFiles:\n"
174 		for f in cl.files:
175 			s += "\t\t" + f + "\n"
176 		return s
177 
178 	def Flush(self, ui, repo):
179 		if self.name == "new":
180 			self.Upload(ui, repo, gofmt_just_warn=True)
181 		dir = CodeReviewDir(ui, repo)
182 		path = dir + '/cl.' + self.name
183 		f = open(path+'!', "w")
184 		f.write(self.DiskText())
185 		f.close()
186 		if sys.platform == "win32" and os.path.isfile(path):
187 			os.remove(path)
188 		os.rename(path+'!', path)
189 		if self.web and not self.copied_from:
190 			EditDesc(self.name, desc=self.desc,
191 				reviewers=JoinComma(self.reviewer), cc=JoinComma(self.cc))
192 
193 	def Delete(self, ui, repo):
194 		dir = CodeReviewDir(ui, repo)
195 		os.unlink(dir + "/cl." + self.name)
196 
197 	def Subject(self):
198 		s = line1(self.desc)
199 		if len(s) > 60:
200 			s = s[0:55] + "..."
201 		if self.name != "new":
202 			s = "code review %s: %s" % (self.name, s)
203 		return s
204 
205 	def Upload(self, ui, repo, send_mail=False, gofmt=True, gofmt_just_warn=False):
206 		if not self.files:
207 			ui.warn("no files in change list\n")
208 		if ui.configbool("codereview", "force_gofmt", True) and gofmt:
209 			CheckGofmt(ui, repo, self.files, just_warn=gofmt_just_warn)
210 		os.chdir(repo.root)
211 		form_fields = [
212 			("content_upload", "1"),
213 			("reviewers", JoinComma(self.reviewer)),
214 			("cc", JoinComma(self.cc)),
215 			("description", self.desc),
216 			("base_hashes", ""),
217 			# Would prefer not to change the subject
218 			# on reupload, but /upload requires it.
219 			("subject", self.Subject()),
220 		]
221 
222 		# NOTE(rsc): This duplicates too much of RealMain,
223 		# but RealMain doesn't have the most reusable interface.
224 		if self.name != "new":
225 			form_fields.append(("issue", self.name))
226 		vcs = None
227 		if self.files:
228 			vcs = GuessVCS(upload_options)
229 			data = vcs.GenerateDiff(self.files)
230 			files = vcs.GetBaseFiles(data)
231 			if len(data) > MAX_UPLOAD_SIZE:
232 				uploaded_diff_file = []
233 				form_fields.append(("separate_patches", "1"))
234 			else:
235 				uploaded_diff_file = [("data", "data.diff", data)]
236 		else:
237 			uploaded_diff_file = [("data", "data.diff", emptydiff)]
238 		ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
239 		response_body = MySend("/upload", body, content_type=ctype)
240 		patchset = None
241 		msg = response_body
242 		lines = msg.splitlines()
243 		if len(lines) >= 2:
244 			msg = lines[0]
245 			patchset = lines[1].strip()
246 			patches = [x.split(" ", 1) for x in lines[2:]]
247 		ui.status(msg + "\n")
248 		if not response_body.startswith("Issue created.") and not response_body.startswith("Issue updated."):
249 			raise util.Abort("failed to update issue: " + response_body)
250 		issue = msg[msg.rfind("/")+1:]
251 		self.name = issue
252 		if not self.url:
253 			self.url = server_url_base + self.name
254 		if not uploaded_diff_file:
255 			patches = UploadSeparatePatches(issue, rpc, patchset, data, upload_options)
256 		if vcs:
257 			vcs.UploadBaseFiles(issue, rpc, patches, patchset, upload_options, files)
258 		if send_mail:
259 			MySend("/" + issue + "/mail", payload="")
260 		self.web = True
261 		self.Flush(ui, repo)
262 		return
263 
264 	def Mail(self, ui,repo):
265 		pmsg = "Hello " + JoinComma(self.reviewer)
266 		if self.cc:
267 			pmsg += " (cc: %s)" % (', '.join(self.cc),)
268 		pmsg += ",\n"
269 		pmsg += "\n"
270 		if not self.mailed:
271 			pmsg += "I'd like you to review this change.\n"
272 		else:
273 			pmsg += "Please take another look.\n"
274 		PostMessage(ui, self.name, pmsg, subject=self.Subject())
275 		self.mailed = True
276 		self.Flush(ui, repo)
277 
278 def GoodCLName(name):
279 	return re.match("^[0-9]+$", name)
280 
281 def ParseCL(text, name):
282 	sname = None
283 	lineno = 0
284 	sections = {
285 		'Author': '',
286 		'Description': '',
287 		'Files': '',
288 		'URL': '',
289 		'Reviewer': '',
290 		'CC': '',
291 		'Mailed': '',
292 	}
293 	for line in text.split('\n'):
294 		lineno += 1
295 		line = line.rstrip()
296 		if line != '' and line[0] == '#':
297 			continue
298 		if line == '' or line[0] == ' ' or line[0] == '\t':
299 			if sname == None and line != '':
300 				return None, lineno, 'text outside section'
301 			if sname != None:
302 				sections[sname] += line + '\n'
303 			continue
304 		p = line.find(':')
305 		if p >= 0:
306 			s, val = line[:p].strip(), line[p+1:].strip()
307 			if s in sections:
308 				sname = s
309 				if val != '':
310 					sections[sname] += val + '\n'
311 				continue
312 		return None, lineno, 'malformed section header'
313 
314 	for k in sections:
315 		sections[k] = StripCommon(sections[k]).rstrip()
316 
317 	cl = CL(name)
318 	if sections['Author']:
319 		cl.copied_from = sections['Author']
320 	cl.desc = sections['Description']
321 	for line in sections['Files'].split('\n'):
322 		i = line.find('#')
323 		if i >= 0:
324 			line = line[0:i].rstrip()
325 		if line == '':
326 			continue
327 		cl.files.append(line)
328 	cl.reviewer = SplitCommaSpace(sections['Reviewer'])
329 	cl.cc = SplitCommaSpace(sections['CC'])
330 	cl.url = sections['URL']
331 	if sections['Mailed'] != 'False':
332 		# Odd default, but avoids spurious mailings when
333 		# reading old CLs that do not have a Mailed: line.
334 		# CLs created with this update will always have 
335 		# Mailed: False on disk.
336 		cl.mailed = True
337 	if cl.desc == '<enter description here>':
338 		cl.desc = ''
339 	return cl, 0, ''
340 
341 def SplitCommaSpace(s):
342 	return re.sub(", *", ",", s).split(",")
343 
344 def CutDomain(s):
345 	i = s.find('@')
346 	if i >= 0:
347 		s = s[0:i]
348 	return s
349 
350 def JoinComma(l):
351 	return ", ".join(l)
352 
353 def ExceptionDetail():
354 	s = str(sys.exc_info()[0])
355 	if s.startswith("<type '") and s.endswith("'>"):
356 		s = s[7:-2]
357 	elif s.startswith("<class '") and s.endswith("'>"):
358 		s = s[8:-2]
359 	arg = str(sys.exc_info()[1])
360 	if len(arg) > 0:
361 		s += ": " + arg
362 	return s
363 
364 def IsLocalCL(ui, repo, name):
365 	return GoodCLName(name) and os.access(CodeReviewDir(ui, repo) + "/cl." + name, 0)
366 
367 # Load CL from disk and/or the web.
368 def LoadCL(ui, repo, name, web=True):
369 	if not GoodCLName(name):
370 		return None, "invalid CL name"
371 	dir = CodeReviewDir(ui, repo)
372 	path = dir + "cl." + name
373 	if os.access(path, 0):
374 		ff = open(path)
375 		text = ff.read()
376 		ff.close()
377 		cl, lineno, err = ParseCL(text, name)
378 		if err != "":
379 			return None, "malformed CL data: "+err
380 		cl.local = True
381 	else:
382 		cl = CL(name)
383 	if web:
384 		try:
385 			f = GetSettings(name)
386 		except:
387 			return None, "cannot load CL %s from code review server: %s" % (name, ExceptionDetail())
388 		if 'reviewers' not in f:
389 			return None, "malformed response loading CL data from code review server"
390 		cl.reviewer = SplitCommaSpace(f['reviewers'])
391 		cl.cc = SplitCommaSpace(f['cc'])
392 		if cl.local and cl.copied_from and cl.desc:
393 			# local copy of CL written by someone else
394 			# and we saved a description.  use that one,
395 			# so that committers can edit the description
396 			# before doing hg submit.
397 			pass
398 		else:
399 			cl.desc = f['description']
400 		cl.url = server_url_base + name
401 		cl.web = True
402 	return cl, ''
403 
404 class LoadCLThread(threading.Thread):
405 	def __init__(self, ui, repo, dir, f, web):
406 		threading.Thread.__init__(self)
407 		self.ui = ui
408 		self.repo = repo
409 		self.dir = dir
410 		self.f = f
411 		self.web = web
412 		self.cl = None
413 	def run(self):
414 		cl, err = LoadCL(self.ui, self.repo, self.f[3:], web=self.web)
415 		if err != '':
416 			self.ui.warn("loading "+self.dir+self.f+": " + err + "\n")
417 			return
418 		self.cl = cl
419 
420 # Load all the CLs from this repository.
421 def LoadAllCL(ui, repo, web=True):
422 	dir = CodeReviewDir(ui, repo)
423 	m = {}
424 	files = [f for f in os.listdir(dir) if f.startswith('cl.')]
425 	if not files:
426 		return m
427 	active = []
428 	first = True
429 	for f in files:
430 		t = LoadCLThread(ui, repo, dir, f, web)
431 		t.start()
432 		if web and first:
433 			# first request: wait in case it needs to authenticate
434 			# otherwise we get lots of user/password prompts
435 			# running in parallel.
436 			t.join()
437 			if t.cl:
438 				m[t.cl.name] = t.cl
439 			first = False
440 		else:
441 			active.append(t)
442 	for t in active:
443 		t.join()
444 		if t.cl:
445 			m[t.cl.name] = t.cl
446 	return m
447 
448 # Find repository root.  On error, ui.warn and return None
449 def RepoDir(ui, repo):
450 	url = repo.url();
451 	if not url.startswith('file:'):
452 		ui.warn("repository %s is not in local file system\n" % (url,))
453 		return None
454 	url = url[5:]
455 	if url.endswith('/'):
456 		url = url[:-1]
457 	return url
458 
459 # Find (or make) code review directory.  On error, ui.warn and return None
460 def CodeReviewDir(ui, repo):
461 	dir = RepoDir(ui, repo)
462 	if dir == None:
463 		return None
464 	dir += '/.hg/codereview/'
465 	if not os.path.isdir(dir):
466 		try:
467 			os.mkdir(dir, 0700)
468 		except:
469 			ui.warn('cannot mkdir %s: %s\n' % (dir, ExceptionDetail()))
470 			return None
471 	return dir
472 
473 # Strip maximal common leading white space prefix from text
474 def StripCommon(text):
475 	ws = None
476 	for line in text.split('\n'):
477 		line = line.rstrip()
478 		if line == '':
479 			continue
480 		white = line[:len(line)-len(line.lstrip())]
481 		if ws == None:
482 			ws = white
483 		else:
484 			common = ''
485 			for i in range(min(len(white), len(ws))+1):
486 				if white[0:i] == ws[0:i]:
487 					common = white[0:i]
488 			ws = common
489 		if ws == '':
490 			break
491 	if ws == None:
492 		return text
493 	t = ''
494 	for line in text.split('\n'):
495 		line = line.rstrip()
496 		if line.startswith(ws):
497 			line = line[len(ws):]
498 		if line == '' and t == '':
499 			continue
500 		t += line + '\n'
501 	while len(t) >= 2 and t[-2:] == '\n\n':
502 		t = t[:-1]
503 	return t
504 
505 # Indent text with indent.
506 def Indent(text, indent):
507 	t = ''
508 	for line in text.split('\n'):
509 		t += indent + line + '\n'
510 	return t
511 
512 # Return the first line of l
513 def line1(text):
514 	return text.split('\n')[0]
515 
516 _change_prolog = """# Change list.
517 # Lines beginning with # are ignored.
518 # Multi-line values should be indented.
519 """
520 
521 #######################################################################
522 # Mercurial helper functions
523 
524 # Return list of changed files in repository that match pats.
525 def ChangedFiles(ui, repo, pats, opts):
526 	# Find list of files being operated on.
527 	matcher = cmdutil.match(repo, pats, opts)
528 	node1, node2 = cmdutil.revpair(repo, None)
529 	modified, added, removed = repo.status(node1, node2, matcher)[:3]
530 	l = modified + added + removed
531 	l.sort()
532 	return l
533 
534 # Return list of changed files in repository that match pats and still exist.
535 def ChangedExistingFiles(ui, repo, pats, opts):
536 	matcher = cmdutil.match(repo, pats, opts)
537 	node1, node2 = cmdutil.revpair(repo, None)
538 	modified, added, _ = repo.status(node1, node2, matcher)[:3]
539 	l = modified + added
540 	l.sort()
541 	return l
542 
543 # Return list of files claimed by existing CLs
544 def TakenFiles(ui, repo):
545 	return Taken(ui, repo).keys()
546 
547 def Taken(ui, repo):
548 	all = LoadAllCL(ui, repo, web=False)
549 	taken = {}
550 	for _, cl in all.items():
551 		for f in cl.files:
552 			taken[f] = cl
553 	return taken
554 
555 # Return list of changed files that are not claimed by other CLs
556 def DefaultFiles(ui, repo, pats, opts):
557 	return Sub(ChangedFiles(ui, repo, pats, opts), TakenFiles(ui, repo))
558 
559 def Sub(l1, l2):
560 	return [l for l in l1 if l not in l2]
561 
562 def Add(l1, l2):
563 	l = l1 + Sub(l2, l1)
564 	l.sort()
565 	return l
566 
567 def Intersect(l1, l2):
568 	return [l for l in l1 if l in l2]
569 
570 def getremote(ui, repo, opts):
571 	# save $http_proxy; creating the HTTP repo object will
572 	# delete it in an attempt to "help"
573 	proxy = os.environ.get('http_proxy')
574 	source, _, _ = hg.parseurl(ui.expandpath("default"), None)
575 	other = hg.repository(cmdutil.remoteui(repo, opts), source)
576 	if proxy is not None:
577 		os.environ['http_proxy'] = proxy
578 	return other
579 
580 def Incoming(ui, repo, opts):
581 	_, incoming, _ = repo.findcommonincoming(getremote(ui, repo, opts))
582 	return incoming
583 
584 def EditCL(ui, repo, cl):
585 	s = cl.EditorText()
586 	while True:
587 		s = ui.edit(s, ui.username())
588 		clx, line, err = ParseCL(s, cl.name)
589 		if err != '':
590 			if ui.prompt("error parsing change list: line %d: %s\nre-edit (y/n)?" % (line, err), ["&yes", "&no"], "y") == "n":
591 				return "change list not modified"
592 			continue
593 		cl.desc = clx.desc;
594 		cl.reviewer = clx.reviewer
595 		cl.cc = clx.cc
596 		cl.files = clx.files
597 		if cl.desc == '':
598 			if ui.prompt("change list should have description\nre-edit (y/n)?", ["&yes", "&no"], "y") != "n":
599 				continue
600 		break
601 	return ""
602 
603 # For use by submit, etc. (NOT by change)
604 # Get change list number or list of files from command line.
605 # If files are given, make a new change list.
606 def CommandLineCL(ui, repo, pats, opts, defaultcc=None):
607 	if len(pats) > 0 and GoodCLName(pats[0]):
608 		if len(pats) != 1:
609 			return None, "cannot specify change number and file names"
610 		if opts.get('message'):
611 			return None, "cannot use -m with existing CL"
612 		cl, err = LoadCL(ui, repo, pats[0], web=True)
613 		if err != "":
614 			return None, err
615 	else:
616 		cl = CL("new")
617 		cl.local = True
618 		cl.files = Sub(ChangedFiles(ui, repo, pats, opts), TakenFiles(ui, repo))
619 		if not cl.files:
620 			return None, "no files changed"
621 	if opts.get('reviewer'):
622 		cl.reviewer = Add(cl.reviewer, SplitCommaSpace(opts.get('reviewer')))
623 	if opts.get('cc'):
624 		cl.cc = Add(cl.cc, SplitCommaSpace(opts.get('cc')))
625 	if defaultcc:
626 		cl.cc = Add(cl.cc, defaultcc)
627 	if cl.name == "new":
628 		if opts.get('message'):
629 			cl.desc = opts.get('message')
630 		else:
631 			err = EditCL(ui, repo, cl)
632 			if err != '':
633 				return None, err
634 	return cl, ""
635 
636 # reposetup replaces cmdutil.match with this wrapper,
637 # which expands the syntax @clnumber to mean the files
638 # in that CL.
639 original_match = None
640 def ReplacementForCmdutilMatch(repo, pats=[], opts={}, globbed=False, default='relpath'):
641 	taken = []
642 	files = []
643 	for p in pats:
644 		if p.startswith('@'):
645 			taken.append(p)
646 			clname = p[1:]
647 			if not GoodCLName(clname):
648 				raise util.Abort("invalid CL name " + clname)
649 			cl, err = LoadCL(repo.ui, repo, clname, web=False)
650 			if err != '':
651 				raise util.Abort("loading CL " + clname + ": " + err)
652 			if cl.files == None:
653 				raise util.Abort("no files in CL " + clname)
654 			files = Add(files, cl.files)
655 	pats = Sub(pats, taken)	+ ['path:'+f for f in files]
656 	return original_match(repo, pats=pats, opts=opts, globbed=globbed, default=default)
657 
658 def RelativePath(path, cwd):
659 	n = len(cwd)
660 	if path.startswith(cwd) and path[n] == '/':
661 		return path[n+1:]
662 	return path
663 
664 # Check that gofmt run on the list of files does not change them
665 def CheckGofmt(ui, repo, files, just_warn=False):
666 	files = [f for f in files if (f.startswith('src/') or f.startswith('test/bench/')) and f.endswith('.go')]
667 	if not files:
668 		return
669 	cwd = os.getcwd()
670 	files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
671 	files = [f for f in files if os.access(f, 0)]
672 	try:
673 		cmd = subprocess.Popen(["gofmt", "-l"] + files, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
674 		cmd.stdin.close()
675 	except:
676 		raise util.Abort("gofmt: " + ExceptionDetail())
677 	data = cmd.stdout.read()
678 	errors = cmd.stderr.read()
679 	cmd.wait()
680 	if len(errors) > 0:
681 		ui.warn("gofmt errors:\n" + errors.rstrip() + "\n")
682 		return
683 	if len(data) > 0:
684 		msg = "gofmt needs to format these files (run hg gofmt):\n" + Indent(data, "\t").rstrip()
685 		if just_warn:
686 			ui.warn("warning: " + msg + "\n")
687 		else:
688 			raise util.Abort(msg)
689 	return
690 
691 #######################################################################
692 # Mercurial commands
693 
694 # every command must take a ui and and repo as arguments.
695 # opts is a dict where you can find other command line flags
696 #
697 # Other parameters are taken in order from items on the command line that
698 # don't start with a dash.  If no default value is given in the parameter list,
699 # they are required.
700 #
701 
702 def change(ui, repo, *pats, **opts):
703 	"""create or edit a change list
704 
705 	Create or edit a change list.
706 	A change list is a group of files to be reviewed and submitted together,
707 	plus a textual description of the change.
708 	Change lists are referred to by simple alphanumeric names.
709 
710 	Changes must be reviewed before they can be submitted.
711 
712 	In the absence of options, the change command opens the
713 	change list for editing in the default editor.
714 
715 	Deleting a change with the -d or -D flag does not affect
716 	the contents of the files listed in that change.  To revert
717 	the files listed in a change, use
718 
719 		hg revert @123456
720 
721 	before running hg change -d 123456.
722 	"""
723 
724 	dirty = {}
725 	if len(pats) > 0 and GoodCLName(pats[0]):
726 		name = pats[0]
727 		if len(pats) != 1:
728 			return "cannot specify CL name and file patterns"
729 		pats = pats[1:]
730 		cl, err = LoadCL(ui, repo, name, web=True)
731 		if err != '':
732 			return err
733 		if not cl.local and (opts["stdin"] or not opts["stdout"]):
734 			return "cannot change non-local CL " + name
735 	else:
736 		name = "new"
737 		cl = CL("new")
738 		dirty[cl] = True
739 		files = ChangedFiles(ui, repo, pats, opts)
740 		taken = TakenFiles(ui, repo)
741 		files = Sub(files, taken)
742 
743 	if opts["delete"] or opts["deletelocal"]:
744 		if opts["delete"] and opts["deletelocal"]:
745 			return "cannot use -d and -D together"
746 		flag = "-d"
747 		if opts["deletelocal"]:
748 			flag = "-D"
749 		if name == "new":
750 			return "cannot use "+flag+" with file patterns"
751 		if opts["stdin"] or opts["stdout"]:
752 			return "cannot use "+flag+" with -i or -o"
753 		if not cl.local:
754 			return "cannot change non-local CL " + name
755 		if opts["delete"]:
756 			if cl.copied_from:
757 				return "original author must delete CL; hg change -D will remove locally"
758 			PostMessage(ui, cl.name, "*** Abandoned ***")
759 			EditDesc(cl.name, closed="checked")
760 		cl.Delete(ui, repo)
761 		return
762 
763 	if opts["stdin"]:
764 		s = sys.stdin.read()
765 		clx, line, err = ParseCL(s, name)
766 		if err != '':
767 			return "error parsing change list: line %d: %s" % (line, err)
768 		if clx.desc is not None:
769 			cl.desc = clx.desc;
770 			dirty[cl] = True
771 		if clx.reviewer is not None:
772 			cl.reviewer = clx.reviewer
773 			dirty[cl] = True
774 		if clx.cc is not None:
775 			cl.cc = clx.cc
776 			dirty[cl] = True
777 		if clx.files is not None:
778 			cl.files = clx.files
779 			dirty[cl] = True
780 
781 	if not opts["stdin"] and not opts["stdout"]:
782 		if name == "new":
783 			cl.files = files
784 		err = EditCL(ui, repo, cl)
785 		if err != "":
786 			return err
787 		dirty[cl] = True
788 
789 	for d, _ in dirty.items():
790 		d.Flush(ui, repo)
791 
792 	if opts["stdout"]:
793 		ui.write(cl.EditorText())
794 	elif name == "new":
795 		if ui.quiet:
796 			ui.write(cl.name)
797 		else:
798 			ui.write("CL created: " + cl.url + "\n")
799 	return
800 
801 def code_login(ui, repo, **opts):
802 	"""log in to code review server
803 
804 	Logs in to the code review server, saving a cookie in
805 	a file in your home directory.
806 	"""
807 	MySend(None)
808 
809 def clpatch(ui, repo, clname, **opts):
810 	"""import a patch from the code review server
811 
812 	Imports a patch from the code review server into the local client.
813 	If the local client has already modified any of the files that the
814 	patch modifies, this command will refuse to apply the patch.
815 
816 	Submitting an imported patch will keep the original author's
817 	name as the Author: line but add your own name to a Committer: line.
818 	"""
819 	cl, patch, err = DownloadCL(ui, repo, clname)
820 	argv = ["hgpatch"]
821 	if opts["no_incoming"]:
822 		argv += ["--checksync=false"]
823 	if err != "":
824 		return err
825 	try:
826 		cmd = subprocess.Popen(argv, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, close_fds=True)
827 	except:
828 		return "hgpatch: " + ExceptionDetail()
829 	if os.fork() == 0:
830 		cmd.stdin.write(patch)
831 		os._exit(0)
832 	cmd.stdin.close()
833 	out = cmd.stdout.read()
834 	if cmd.wait() != 0 and not opts["ignore_hgpatch_failure"]:
835 		return "hgpatch failed"
836 	cl.local = True
837 	cl.files = out.strip().split()
838 	files = ChangedFiles(ui, repo, [], opts)
839 	extra = Sub(cl.files, files)
840 	if extra:
841 		ui.warn("warning: these files were listed in the patch but not changed:\n\t" + "\n\t".join(extra) + "\n")
842 	cl.Flush(ui, repo)
843 	ui.write(cl.PendingText() + "\n")
844 
845 def download(ui, repo, clname, **opts):
846 	"""download a change from the code review server
847 
848 	Download prints a description of the given change list
849 	followed by its diff, downloaded from the code review server.
850 	"""
851 	cl, patch, err = DownloadCL(ui, repo, clname)
852 	if err != "":
853 		return err
854 	ui.write(cl.EditorText() + "\n")
855 	ui.write(patch + "\n")
856 	return
857 
858 def file(ui, repo, clname, pat, *pats, **opts):
859 	"""assign files to or remove files from a change list
860 
861 	Assign files to or (with -d) remove files from a change list.
862 
863 	The -d option only removes files from the change list.
864 	It does not edit them or remove them from the repository.
865 	"""
866 	pats = tuple([pat] + list(pats))
867 	if not GoodCLName(clname):
868 		return "invalid CL name " + clname
869 
870 	dirty = {}
871 	cl, err = LoadCL(ui, repo, clname, web=False)
872 	if err != '':
873 		return err
874 	if not cl.local:
875 		return "cannot change non-local CL " + clname
876 
877 	files = ChangedFiles(ui, repo, pats, opts)
878 
879 	if opts["delete"]:
880 		oldfiles = Intersect(files, cl.files)
881 		if oldfiles:
882 			if not ui.quiet:
883 				ui.status("# Removing files from CL.  To undo:\n")
884 				ui.status("#	cd %s\n" % (repo.root))
885 				for f in oldfiles:
886 					ui.status("#	hg file %s %s\n" % (cl.name, f))
887 			cl.files = Sub(cl.files, oldfiles)
888 			cl.Flush(ui, repo)
889 		else:
890 			ui.status("no such files in CL")
891 		return
892 
893 	if not files:
894 		return "no such modified files"
895 
896 	files = Sub(files, cl.files)
897 	taken = Taken(ui, repo)
898 	warned = False
899 	for f in files:
900 		if f in taken:
901 			if not warned and not ui.quiet:
902 				ui.status("# Taking files from other CLs.  To undo:\n")
903 				ui.status("#	cd %s\n" % (repo.root))
904 				warned = True
905 			ocl = taken[f]
906 			if not ui.quiet:
907 				ui.status("#	hg file %s %s\n" % (ocl.name, f))
908 			if ocl not in dirty:
909 				ocl.files = Sub(ocl.files, files)
910 				dirty[ocl] = True
911 	cl.files = Add(cl.files, files)
912 	dirty[cl] = True
913 	for d, _ in dirty.items():
914 		d.Flush(ui, repo)
915 	return
916 
917 def gofmt(ui, repo, *pats, **opts):
918 	"""apply gofmt to modified files
919 
920 	Applies gofmt to the modified files in the repository that match
921 	the given patterns.
922 	"""
923 	files = ChangedExistingFiles(ui, repo, pats, opts)
924 	files = [f for f in files if f.endswith(".go")]
925 	if not files:
926 		return "no modified go files"
927 	cwd = os.getcwd()
928 	files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
929 	try:
930 		cmd = ["gofmt", "-l"]
931 		if not opts["list"]:
932 			cmd += ["-w"]
933 		if os.spawnvp(os.P_WAIT, "gofmt", cmd + files) != 0:
934 			raise util.Abort("gofmt did not exit cleanly")
935 	except error.Abort, e:
936 		raise
937 	except:
938 		raise util.Abort("gofmt: " + ExceptionDetail())
939 	return
940 
941 def mail(ui, repo, *pats, **opts):
942 	"""mail a change for review
943 
944 	Uploads a patch to the code review server and then sends mail
945 	to the reviewer and CC list asking for a review.
946 	"""
947 	cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
948 	if err != "":
949 		return err
950 	cl.Upload(ui, repo, gofmt_just_warn=True)
951 	if not cl.reviewer and not cl.cc:
952 		return "no reviewers listed in CL"
953 	cl.Mail(ui, repo)
954 
955 def nocommit(ui, repo, *pats, **opts):
956 	"""(disabled when using this extension)"""
957 	return "The codereview extension is enabled; do not use commit."
958 
959 def pending(ui, repo, *pats, **opts):
960 	"""show pending changes
961 
962 	Lists pending changes followed by a list of unassigned but modified files.
963 	"""
964 	m = LoadAllCL(ui, repo, web=True)
965 	names = m.keys()
966 	names.sort()
967 	for name in names:
968 		cl = m[name]
969 		ui.write(cl.PendingText() + "\n")
970 
971 	files = DefaultFiles(ui, repo, [], opts)
972 	if len(files) > 0:
973 		s = "Changed files not in any CL:\n"
974 		for f in files:
975 			s += "\t" + f + "\n"
976 		ui.write(s)
977 
978 def reposetup(ui, repo):
979 	global original_match
980 	if original_match is None:
981 		original_match = cmdutil.match
982 		cmdutil.match = ReplacementForCmdutilMatch
983 		RietveldSetup(ui, repo)
984 
985 def CheckContributor(ui, repo, user=None):
986 	if not user:
987 		user = ui.config("ui", "username")
988 		if not user:
989 			raise util.Abort("[ui] username is not configured in .hgrc")
990 	_, userline = FindContributor(ui, repo, user, warn=False)
991 	if not userline:
992 		raise util.Abort("cannot find %s in CONTRIBUTORS" % (user,))
993 	return userline
994 
995 def FindContributor(ui, repo, user, warn=True):
996 	try:
997 		f = open(repo.root + '/CONTRIBUTORS', 'r')
998 	except:
999 		raise util.Abort("cannot open %s: %s" % (repo.root+'/CONTRIBUTORS', ExceptionDetail()))
1000 	for line in f.readlines():
1001 		line = line.rstrip()
1002 		if line.startswith('#'):
1003 			continue
1004 		match = re.match(r"(.*) <(.*)>", line)
1005 		if not match:
1006 			continue
1007 		if line == user or match.group(2).lower() == user.lower():
1008 			return match.group(2), line
1009 	if warn:
1010 		ui.warn("warning: cannot find %s in CONTRIBUTORS\n" % (user,))
1011 	return None, None
1012 
1013 def submit(ui, repo, *pats, **opts):
1014 	"""submit change to remote repository
1015 
1016 	Submits change to remote repository.
1017 	Bails out if the local repository is not in sync with the remote one.
1018 	"""
1019 	repo.ui.quiet = True
1020 	if not opts["no_incoming"] and Incoming(ui, repo, opts):
1021 		return "local repository out of date; must sync before submit"
1022 
1023 	cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
1024 	if err != "":
1025 		return err
1026 
1027 	user = None
1028 	if cl.copied_from:
1029 		user = cl.copied_from
1030 	userline = CheckContributor(ui, repo, user)
1031 
1032 	about = ""
1033 	if cl.reviewer:
1034 		about += "R=" + JoinComma([CutDomain(s) for s in cl.reviewer]) + "\n"
1035 	if opts.get('tbr'):
1036 		tbr = SplitCommaSpace(opts.get('tbr'))
1037 		cl.reviewer = Add(cl.reviewer, tbr)
1038 		about += "TBR=" + JoinComma([CutDomain(s) for s in tbr]) + "\n"
1039 	if cl.cc:
1040 		about += "CC=" + JoinComma([CutDomain(s) for s in cl.cc]) + "\n"
1041 
1042 	if not cl.reviewer:
1043 		return "no reviewers listed in CL"
1044 
1045 	if not cl.local:
1046 		return "cannot submit non-local CL"
1047 
1048 	# upload, to sync current patch and also get change number if CL is new.
1049 	if not cl.copied_from:
1050 		cl.Upload(ui, repo, gofmt_just_warn=True)
1051 
1052 	# check gofmt for real; allowed upload to warn in order to save CL.
1053 	cl.Flush(ui, repo)
1054 	CheckGofmt(ui, repo, cl.files)
1055 
1056 	about += "%s%s\n" % (server_url_base, cl.name)
1057 
1058 	if cl.copied_from:
1059 		about += "\nCommitter: " + CheckContributor(ui, repo, None) + "\n"
1060 
1061 	if not cl.mailed and not cl.copied_from:		# in case this is TBR
1062 		cl.Mail(ui, repo)
1063 
1064 	# submit changes locally
1065 	date = opts.get('date')
1066 	if date:
1067 		opts['date'] = util.parsedate(date)
1068 	opts['message'] = cl.desc.rstrip() + "\n\n" + about
1069 
1070 	if opts['dryrun']:
1071 		print "NOT SUBMITTING:"
1072 		print "User: ", userline
1073 		print "Message:"
1074 		print Indent(opts['message'], "\t")
1075 		print "Files:"
1076 		print Indent('\n'.join(cl.files), "\t")
1077 		return "dry run; not submitted"
1078 
1079 	m = match.exact(repo.root, repo.getcwd(), cl.files)
1080 	node = repo.commit(opts['message'], userline, opts.get('date'), m)
1081 	if not node:
1082 		return "nothing changed"
1083 
1084 	# push to remote; if it fails for any reason, roll back
1085 	try:
1086 		log = repo.changelog
1087 		rev = log.rev(node)
1088 		parents = log.parentrevs(rev)
1089 		if (rev-1 not in parents and
1090 				(parents == (nullrev, nullrev) or
1091 				len(log.heads(log.node(parents[0]))) > 1 and
1092 				(parents[1] == nullrev or len(log.heads(log.node(parents[1]))) > 1))):
1093 			# created new head
1094 			raise util.Abort("local repository out of date; must sync before submit")
1095 
1096 		# push changes to remote.
1097 		# if it works, we're committed.
1098 		# if not, roll back
1099 		other = getremote(ui, repo, opts)
1100 		r = repo.push(other, False, None)
1101 		if r == 0:
1102 			raise util.Abort("local repository out of date; must sync before submit")
1103 	except:
1104 		repo.rollback()
1105 		raise
1106 
1107 	# we're committed. upload final patch, close review, add commit message
1108 	changeURL = short(node)
1109 	url = other.url()
1110 	m = re.match("^https?://([^@/]+@)?([^.]+)\.googlecode\.com/hg/", url)
1111 	if m:
1112 		changeURL = "http://code.google.com/p/%s/source/detail?r=%s" % (m.group(2), changeURL)
1113 	else:
1114 		print >>sys.stderr, "URL: ", url
1115 	pmsg = "*** Submitted as " + changeURL + " ***\n\n" + opts['message']
1116 
1117 	# When posting, move reviewers to CC line,
1118 	# so that the issue stops showing up in their "My Issues" page.
1119 	PostMessage(ui, cl.name, pmsg, reviewers="", cc=JoinComma(cl.reviewer+cl.cc))
1120 
1121 	if not cl.copied_from:
1122 		EditDesc(cl.name, closed="checked")
1123 	cl.Delete(ui, repo)
1124 
1125 def sync(ui, repo, **opts):
1126 	"""synchronize with remote repository
1127 
1128 	Incorporates recent changes from the remote repository
1129 	into the local repository.
1130 	"""
1131 	if not opts["local"]:
1132 		ui.status = sync_note
1133 		ui.note = sync_note
1134 		other = getremote(ui, repo, opts)
1135 		modheads = repo.pull(other)
1136 		err = commands.postincoming(ui, repo, modheads, True, "tip")
1137 		if err:
1138 			return err
1139 	commands.update(ui, repo)
1140 	sync_changes(ui, repo)
1141 
1142 def sync_note(msg):
1143 	# we run sync (pull -u) in verbose mode to get the
1144 	# list of files being updated, but that drags along
1145 	# a bunch of messages we don't care about.
1146 	# omit them.
1147 	if msg == 'resolving manifests\n':
1148 		return
1149 	if msg == 'searching for changes\n':
1150 		return
1151 	if msg == "couldn't find merge tool hgmerge\n":
1152 		return
1153 	sys.stdout.write(msg)
1154 
1155 def sync_changes(ui, repo):
1156 	# Look through recent change log descriptions to find
1157 	# potential references to http://.*/our-CL-number.
1158 	# Double-check them by looking at the Rietveld log.
1159 	def Rev(rev):
1160 		desc = repo[rev].description().strip()
1161 		for clname in re.findall('(?m)^http://(?:[^\n]+)/([0-9]+)$', desc):
1162 			if IsLocalCL(ui, repo, clname) and IsRietveldSubmitted(ui, clname, repo[rev].hex()):
1163 				ui.warn("CL %s submitted as %s; closing\n" % (clname, repo[rev]))
1164 				cl, err = LoadCL(ui, repo, clname, web=False)
1165 				if err != "":
1166 					ui.warn("loading CL %s: %s\n" % (clname, err))
1167 					continue
1168 				if not cl.copied_from:
1169 					EditDesc(cl.name, closed="checked")
1170 				cl.Delete(ui, repo)
1171 
1172 	if hgversion < '1.4':
1173 		get = util.cachefunc(lambda r: repo[r].changeset())
1174 		changeiter, matchfn = cmdutil.walkchangerevs(ui, repo, [], get, {'rev': None})
1175 		n = 0
1176 		for st, rev, fns in changeiter:
1177 			if st != 'iter':
1178 				continue
1179 			n += 1
1180 			if n > 100:
1181 				break
1182 			Rev(rev)
1183 	else:
1184 		matchfn = cmdutil.match(repo, [], {'rev': None})
1185 		def prep(ctx, fns):
1186 			pass
1187 		for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev': None}, prep):
1188 			Rev(ctx.rev())
1189 
1190 	# Remove files that are not modified from the CLs in which they appear.
1191 	all = LoadAllCL(ui, repo, web=False)
1192 	changed = ChangedFiles(ui, repo, [], {})
1193 	for _, cl in all.items():
1194 		extra = Sub(cl.files, changed)
1195 		if extra:
1196 			ui.warn("Removing unmodified files from CL %s:\n" % (cl.name,))
1197 			for f in extra:
1198 				ui.warn("\t%s\n" % (f,))
1199 			cl.files = Sub(cl.files, extra)
1200 			cl.Flush(ui, repo)
1201 		if not cl.files:
1202 			ui.warn("CL %s has no files; suggest hg change -d %s\n" % (cl.name, cl.name))
1203 	return
1204 
1205 def uisetup(ui):
1206 	if "^commit|ci" in commands.table:
1207 		commands.table["^commit|ci"] = (nocommit, [], "")
1208 
1209 def upload(ui, repo, name, **opts):
1210 	"""upload diffs to the code review server
1211 
1212 	Uploads the current modifications for a given change to the server.
1213 	"""
1214 	repo.ui.quiet = True
1215 	cl, err = LoadCL(ui, repo, name, web=True)
1216 	if err != "":
1217 		return err
1218 	if not cl.local:
1219 		return "cannot upload non-local change"
1220 	cl.Upload(ui, repo)
1221 	print "%s%s\n" % (server_url_base, cl.name)
1222 	return
1223 
1224 review_opts = [
1225 	('r', 'reviewer', '', 'add reviewer'),
1226 	('', 'cc', '', 'add cc'),
1227 	('', 'tbr', '', 'add future reviewer'),
1228 	('m', 'message', '', 'change description (for new change)'),
1229 ]
1230 
1231 cmdtable = {
1232 	# The ^ means to show this command in the help text that
1233 	# is printed when running hg with no arguments.
1234 	"^change": (
1235 		change,
1236 		[
1237 			('d', 'delete', None, 'delete existing change list'),
1238 			('D', 'deletelocal', None, 'delete locally, but do not change CL on server'),
1239 			('i', 'stdin', None, 'read change list from standard input'),
1240 			('o', 'stdout', None, 'print change list to standard output'),
1241 		],
1242 		"[-d | -D] [-i] [-o] change# or FILE ..."
1243 	),
1244 	"^clpatch": (
1245 		clpatch,
1246 		[
1247 			('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
1248 			('', 'no_incoming', None, 'disable check for incoming changes'),
1249 		],
1250 		"change#"
1251 	),
1252 	# Would prefer to call this codereview-login, but then
1253 	# hg help codereview prints the help for this command
1254 	# instead of the help for the extension.
1255 	"code-login": (
1256 		code_login,
1257 		[],
1258 		"",
1259 	),
1260 	"commit|ci": (
1261 		nocommit,
1262 		[],
1263 		"",
1264 	),
1265 	"^download": (
1266 		download,
1267 		[],
1268 		"change#"
1269 	),
1270 	"^file": (
1271 		file,
1272 		[
1273 			('d', 'delete', None, 'delete files from change list (but not repository)'),
1274 		],
1275 		"[-d] change# FILE ..."
1276 	),
1277 	"^gofmt": (
1278 		gofmt,
1279 		[
1280 			('l', 'list', None, 'list files that would change, but do not edit them'),
1281 		],
1282 		"FILE ..."
1283 	),
1284 	"^pending|p": (
1285 		pending,
1286 		[],
1287 		"[FILE ...]"
1288 	),
1289 	"^mail": (
1290 		mail,
1291 		review_opts + [
1292 		] + commands.walkopts,
1293 		"[-r reviewer] [--cc cc] [change# | file ...]"
1294 	),
1295 	"^submit": (
1296 		submit,
1297 		review_opts + [
1298 			('', 'no_incoming', None, 'disable initial incoming check (for testing)'),
1299 			('n', 'dryrun', None, 'make change only locally (for testing)'),
1300 		] + commands.walkopts + commands.commitopts + commands.commitopts2,
1301 		"[-r reviewer] [--cc cc] [change# | file ...]"
1302 	),
1303 	"^sync": (
1304 		sync,
1305 		[
1306 			('', 'local', None, 'do not pull changes from remote repository')
1307 		],
1308 		"[--local]",
1309 	),
1310 	"^upload": (
1311 		upload,
1312 		[],
1313 		"change#"
1314 	),
1315 }
1316 
1317 
1318 #######################################################################
1319 # Wrappers around upload.py for interacting with Rietveld
1320 
1321 # HTML form parser
1322 class FormParser(HTMLParser):
1323 	def __init__(self):
1324 		self.map = {}
1325 		self.curtag = None
1326 		self.curdata = None
1327 		HTMLParser.__init__(self)
1328 	def handle_starttag(self, tag, attrs):
1329 		if tag == "input":
1330 			key = None
1331 			value = ''
1332 			for a in attrs:
1333 				if a[0] == 'name':
1334 					key = a[1]
1335 				if a[0] == 'value':
1336 					value = a[1]
1337 			if key is not None:
1338 				self.map[key] = value
1339 		if tag == "textarea":
1340 			key = None
1341 			for a in attrs:
1342 				if a[0] == 'name':
1343 					key = a[1]
1344 			if key is not None:
1345 				self.curtag = key
1346 				self.curdata = ''
1347 	def handle_endtag(self, tag):
1348 		if tag == "textarea" and self.curtag is not None:
1349 			self.map[self.curtag] = self.curdata
1350 			self.curtag = None
1351 			self.curdata = None
1352 	def handle_charref(self, name):
1353 		self.handle_data(unichr(int(name)))
1354 	def handle_entityref(self, name):
1355 		import htmlentitydefs
1356 		if name in htmlentitydefs.entitydefs:
1357 			self.handle_data(htmlentitydefs.entitydefs[name])
1358 		else:
1359 			self.handle_data("&" + name + ";")
1360 	def handle_data(self, data):
1361 		if self.curdata is not None:
1362 			self.curdata += data.decode("utf-8").encode("utf-8")
1363 
1364 # XML parser
1365 def XMLGet(ui, path):
1366 	try:
1367 		data = MySend(path, force_auth=False);
1368 	except:
1369 		ui.warn("XMLGet %s: %s\n" % (path, ExceptionDetail()))
1370 		return None
1371 	return ET.XML(data)
1372 
1373 def IsRietveldSubmitted(ui, clname, hex):
1374 	feed = XMLGet(ui, "/rss/issue/" + clname)
1375 	if feed is None:
1376 		return False
1377 	for sum in feed.findall("{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}summary"):
1378 		text = sum.findtext("", None).strip()
1379 		m = re.match('\*\*\* Submitted as [^*]*?([0-9a-f]+) \*\*\*', text)
1380 		if m is not None and len(m.group(1)) >= 8 and hex.startswith(m.group(1)):
1381 			return True
1382 	return False
1383 
1384 def DownloadCL(ui, repo, clname):
1385 	cl, err = LoadCL(ui, repo, clname)
1386 	if err != "":
1387 		return None, None, "error loading CL %s: %s" % (clname, ExceptionDetail())
1388 
1389 	# Grab RSS feed to learn about CL
1390 	feed = XMLGet(ui, "/rss/issue/" + clname)
1391 	if feed is None:
1392 		return None, None, "cannot download CL"
1393 
1394 	# Find most recent diff
1395 	diff = None
1396 	prefix = 'http://' + server + '/'
1397 	for link in feed.findall("{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}link"):
1398 		if link.get('rel') != 'alternate':
1399 			continue
1400 		text = link.get('href')
1401 		if not text.startswith(prefix) or not text.endswith('.diff'):
1402 			continue
1403 		diff = text[len(prefix)-1:]
1404 	if diff is None:
1405 		return None, None, "CL has no diff"
1406 	diffdata = MySend(diff, force_auth=False)
1407 
1408 	# Find author - first entry will be author who created CL.
1409 	nick = None
1410 	for author in feed.findall("{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}author/{http://www.w3.org/2005/Atom}name"):
1411 		nick = author.findtext("", None).strip()
1412 		break
1413 	if not nick:
1414 		return None, None, "CL has no author"
1415 
1416 	# The author is just a nickname: get the real email address.
1417 	try:
1418 		# want URL-encoded nick, but without a=, and rietveld rejects + for %20.
1419 		url = "/user_popup/" + urllib.urlencode({"a": nick})[2:].replace("+", "%20")
1420 		data = MySend(url, force_auth=False)
1421 	except:
1422 		ui.warn("error looking up %s: %s\n" % (nick, ExceptionDetail()))
1423 		cl.copied_from = nick+"@needtofix"
1424 		return cl, diffdata, ""
1425 	match = re.match(r"<b>(.*) \((.*)\)</b>", data)
1426 	if not match:
1427 		return None, None, "error looking up %s: cannot parse result %s" % (nick, repr(data))
1428 	if match.group(1) != nick and match.group(2) != nick:
1429 		return None, None, "error looking up %s: got info for %s, %s" % (nick, match.group(1), match.group(2))
1430 	email = match.group(1)
1431 
1432 	# Print warning if email is not in CONTRIBUTORS file.
1433 	FindContributor(ui, repo, email)
1434 	cl.copied_from = email
1435 
1436 	return cl, diffdata, ""
1437 
1438 def MySend(request_path, payload=None,
1439            content_type="application/octet-stream",
1440            timeout=None, force_auth=True,
1441            **kwargs):
1442      """Run MySend1 maybe twice, because Rietveld is unreliable."""
1443      try:
1444          return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
1445      except Exception, e:
1446          if type(e) == urllib2.HTTPError and e.code == 403:	# forbidden, it happens
1447          	raise
1448          print >>sys.stderr, "Loading "+request_path+": "+ExceptionDetail()+"; trying again in 2 seconds."
1449      time.sleep(2)
1450      return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
1451 
1452 
1453 # Like upload.py Send but only authenticates when the
1454 # redirect is to www.google.com/accounts.  This keeps
1455 # unnecessary redirects from happening during testing.
1456 def MySend1(request_path, payload=None,
1457            content_type="application/octet-stream",
1458            timeout=None, force_auth=True,
1459            **kwargs):
1460     """Sends an RPC and returns the response.
1461 
1462     Args:
1463       request_path: The path to send the request to, eg /api/appversion/create.
1464       payload: The body of the request, or None to send an empty request.
1465       content_type: The Content-Type header to use.
1466       timeout: timeout in seconds; default None i.e. no timeout.
1467         (Note: for large requests on OS X, the timeout doesn't work right.)
1468       kwargs: Any keyword arguments are converted into query string parameters.
1469 
1470     Returns:
1471       The response body, as a string.
1472     """
1473     # TODO: Don't require authentication.  Let the server say
1474     # whether it is necessary.
1475     global rpc
1476     if rpc == None:
1477     	rpc = GetRpcServer(upload_options)
1478     self = rpc
1479     if not self.authenticated and force_auth:
1480       self._Authenticate()
1481     if request_path is None:
1482       return
1483 
1484     old_timeout = socket.getdefaulttimeout()
1485     socket.setdefaulttimeout(timeout)
1486     try:
1487       tries = 0
1488       while True:
1489         tries += 1
1490         args = dict(kwargs)
1491         url = "http://%s%s" % (self.host, request_path)
1492         if args:
1493           url += "?" + urllib.urlencode(args)
1494         req = self._CreateRequest(url=url, data=payload)
1495         req.add_header("Content-Type", content_type)
1496         try:
1497           f = self.opener.open(req)
1498           response = f.read()
1499           f.close()
1500           # Translate \r\n into \n, because Rietveld doesn't.
1501           response = response.replace('\r\n', '\n')
1502           return response
1503         except urllib2.HTTPError, e:
1504           if tries > 3:
1505             raise
1506           elif e.code == 401:
1507             self._Authenticate()
1508           elif e.code == 302:
1509             loc = e.info()["location"]
1510             if not loc.startswith('https://www.google.com/a') or loc.find('/ServiceLogin') < 0:
1511               return ''
1512             self._Authenticate()
1513           else:
1514             raise
1515     finally:
1516       socket.setdefaulttimeout(old_timeout)
1517 
1518 def GetForm(url):
1519 	f = FormParser()
1520 	f.feed(MySend(url))
1521 	f.close()
1522 	for k,v in f.map.items():
1523 		f.map[k] = v.replace("\r\n", "\n");
1524 	return f.map
1525 
1526 # Fetch the settings for the CL, like reviewer and CC list, by
1527 # scraping the Rietveld editing forms.
1528 def GetSettings(issue):
1529 	# The /issue/edit page has everything but only the
1530 	# CL owner is allowed to fetch it (and submit it).
1531 	f = None
1532 	try:
1533 		f = GetForm("/" + issue + "/edit")
1534 	except:
1535 		pass
1536 	if not f or 'reviewers' not in f:
1537 		# Maybe we're not the CL owner.  Fall back to the
1538 		# /publish page, which has the reviewer and CC lists,
1539 		# and then fetch the description separately.
1540 		f = GetForm("/" + issue + "/publish")
1541 		f['description'] = MySend("/"+issue+"/description", force_auth=False)
1542 	return f
1543 
1544 def EditDesc(issue, subject=None, desc=None, reviewers=None, cc=None, closed=None):
1545 	form_fields = GetForm("/" + issue + "/edit")
1546 	if subject is not None:
1547 		form_fields['subject'] = subject
1548 	if desc is not None:
1549 		form_fields['description'] = desc
1550 	if reviewers is not None:
1551 		form_fields['reviewers'] = reviewers
1552 	if cc is not None:
1553 		form_fields['cc'] = cc
1554 	if closed is not None:
1555 		form_fields['closed'] = closed
1556 	ctype, body = EncodeMultipartFormData(form_fields.items(), [])
1557 	response = MySend("/" + issue + "/edit", body, content_type=ctype)
1558 	if response != "":
1559 		print >>sys.stderr, "Error editing description:\n" + "Sent form: \n", form_fields, "\n", response
1560 		sys.exit(2)
1561 
1562 def PostMessage(ui, issue, message, reviewers=None, cc=None, send_mail=True, subject=None):
1563 	form_fields = GetForm("/" + issue + "/publish")
1564 	if reviewers is not None:
1565 		form_fields['reviewers'] = reviewers
1566 	if cc is not None:
1567 		form_fields['cc'] = cc
1568 	if send_mail:
1569 		form_fields['send_mail'] = "checked"
1570 	else:
1571 		del form_fields['send_mail']
1572 	if subject is not None:
1573 		form_fields['subject'] = subject
1574 	form_fields['message'] = message
1575 	
1576 	form_fields['message_only'] = '1'	# Don't include draft comments
1577 	if reviewers is not None or cc is not None:
1578 		form_fields['message_only'] = ''	# Must set '' in order to override cc/reviewer
1579 	ctype = "applications/x-www-form-urlencoded"
1580 	body = urllib.urlencode(form_fields)
1581 	response = MySend("/" + issue + "/publish", body, content_type=ctype)
1582 	if response != "":
1583 		print response
1584 		sys.exit(2)
1585 
1586 class opt(object):
1587 	pass
1588 
1589 def RietveldSetup(ui, repo):
1590 	global defaultcc, upload_options, rpc, server, server_url_base, force_google_account, verbosity
1591 
1592 	# Read repository-specific options from lib/codereview/codereview.cfg
1593 	try:
1594 		f = open(repo.root + '/lib/codereview/codereview.cfg')
1595 		for line in f:
1596 			if line.startswith('defaultcc: '):
1597 				defaultcc = SplitCommaSpace(line[10:])
1598 	except:
1599 		pass
1600 
1601 	# TODO(rsc): If the repository config has no codereview section,
1602 	# do not enable the extension.  This allows users to
1603 	# put the extension in their global .hgrc but only
1604 	# enable it for some repositories.
1605 	# if not ui.has_section("codereview"):
1606 	# 	cmdtable = {}
1607 	# 	return
1608 
1609 	if not ui.verbose:
1610 		verbosity = 0
1611 
1612 	# Config options.
1613 	x = ui.config("codereview", "server")
1614 	if x is not None:
1615 		server = x
1616 
1617 	# TODO(rsc): Take from ui.username?
1618 	email = None
1619 	x = ui.config("codereview", "email")
1620 	if x is not None:
1621 		email = x
1622 
1623 	server_url_base = "http://" + server + "/"
1624 
1625 	testing = ui.config("codereview", "testing")
1626 	force_google_account = ui.configbool("codereview", "force_google_account", False)
1627 
1628 	upload_options = opt()
1629 	upload_options.email = email
1630 	upload_options.host = None
1631 	upload_options.verbose = 0
1632 	upload_options.description = None
1633 	upload_options.description_file = None
1634 	upload_options.reviewers = None
1635 	upload_options.cc = None
1636 	upload_options.message = None
1637 	upload_options.issue = None
1638 	upload_options.download_base = False
1639 	upload_options.revision = None
1640 	upload_options.send_mail = False
1641 	upload_options.vcs = None
1642 	upload_options.server = server
1643 	upload_options.save_cookies = True
1644 
1645 	if testing:
1646 		upload_options.save_cookies = False
1647 		upload_options.email = "test@example.com"
1648 
1649 	rpc = None
1650 
1651 #######################################################################
1652 # We keep a full copy of upload.py here to avoid import path hell.
1653 # It would be nice if hg added the hg repository root
1654 # to the default PYTHONPATH.
1655 
1656 # Edit .+2,<hget http://codereview.appspot.com/static/upload.py
1657 
1658 #!/usr/bin/env python
1659 #
1660 # Copyright 2007 Google Inc.
1661 #
1662 # Licensed under the Apache License, Version 2.0 (the "License");
1663 # you may not use this file except in compliance with the License.
1664 # You may obtain a copy of the License at
1665 #
1666 #     http://www.apache.org/licenses/LICENSE-2.0
1667 #
1668 # Unless required by applicable law or agreed to in writing, software
1669 # distributed under the License is distributed on an "AS IS" BASIS,
1670 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1671 # See the License for the specific language governing permissions and
1672 # limitations under the License.
1673 
1674 """Tool for uploading diffs from a version control system to the codereview app.
1675 
1676 Usage summary: upload.py [options] [-- diff_options]
1677 
1678 Diff options are passed to the diff command of the underlying system.
1679 
1680 Supported version control systems:
1681   Git
1682   Mercurial
1683   Subversion
1684 
1685 It is important for Git/Mercurial users to specify a tree/node/branch to diff
1686 against by using the '--rev' option.
1687 """
1688 # This code is derived from appcfg.py in the App Engine SDK (open source),
1689 # and from ASPN recipe #146306.
1690 
1691 import cookielib
1692 import getpass
1693 import logging
1694 import mimetypes
1695 import optparse
1696 import os
1697 import re
1698 import socket
1699 import subprocess
1700 import sys
1701 import urllib
1702 import urllib2
1703 import urlparse
1704 
1705 # The md5 module was deprecated in Python 2.5.
1706 try:
1707   from hashlib import md5
1708 except ImportError:
1709   from md5 import md5
1710 
1711 try:
1712   import readline
1713 except ImportError:
1714   pass
1715 
1716 # The logging verbosity:
1717 #  0: Errors only.
1718 #  1: Status messages.
1719 #  2: Info logs.
1720 #  3: Debug logs.
1721 verbosity = 1
1722 
1723 # Max size of patch or base file.
1724 MAX_UPLOAD_SIZE = 900 * 1024
1725 
1726 # Constants for version control names.  Used by GuessVCSName.
1727 VCS_GIT = "Git"
1728 VCS_MERCURIAL = "Mercurial"
1729 VCS_SUBVERSION = "Subversion"
1730 VCS_UNKNOWN = "Unknown"
1731 
1732 # whitelist for non-binary filetypes which do not start with "text/"
1733 # .mm (Objective-C) shows up as application/x-freemind on my Linux box.
1734 TEXT_MIMETYPES = ['application/javascript', 'application/x-javascript',
1735                   'application/x-freemind']
1736 
1737 VCS_ABBREVIATIONS = {
1738   VCS_MERCURIAL.lower(): VCS_MERCURIAL,
1739   "hg": VCS_MERCURIAL,
1740   VCS_SUBVERSION.lower(): VCS_SUBVERSION,
1741   "svn": VCS_SUBVERSION,
1742   VCS_GIT.lower(): VCS_GIT,
1743 }
1744 
1745 
1746 def GetEmail(prompt):
1747   """Prompts the user for their email address and returns it.
1748 
1749   The last used email address is saved to a file and offered up as a suggestion
1750   to the user. If the user presses enter without typing in anything the last
1751   used email address is used. If the user enters a new address, it is saved
1752   for next time we prompt.
1753 
1754   """
1755   last_email_file_name = os.path.expanduser("~/.last_codereview_email_address")
1756   last_email = ""
1757   if os.path.exists(last_email_file_name):
1758     try:
1759       last_email_file = open(last_email_file_name, "r")
1760       last_email = last_email_file.readline().strip("\n")
1761       last_email_file.close()
1762       prompt += " [%s]" % last_email
1763     except IOError, e:
1764       pass
1765   email = raw_input(prompt + ": ").strip()
1766   if email:
1767     try:
1768       last_email_file = open(last_email_file_name, "w")
1769       last_email_file.write(email)
1770       last_email_file.close()
1771     except IOError, e:
1772       pass
1773   else:
1774     email = last_email
1775   return email
1776 
1777 
1778 def StatusUpdate(msg):
1779   """Print a status message to stdout.
1780 
1781   If 'verbosity' is greater than 0, print the message.
1782 
1783   Args:
1784     msg: The string to print.
1785   """
1786   if verbosity > 0:
1787     print msg
1788 
1789 
1790 def ErrorExit(msg):
1791   """Print an error message to stderr and exit."""
1792   print >>sys.stderr, msg
1793   sys.exit(1)
1794 
1795 
1796 class ClientLoginError(urllib2.HTTPError):
1797   """Raised to indicate there was an error authenticating with ClientLogin."""
1798 
1799   def __init__(self, url, code, msg, headers, args):
1800     urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
1801     self.args = args
1802     self.reason = args["Error"]
1803 
1804 
1805 class AbstractRpcServer(object):
1806   """Provides a common interface for a simple RPC server."""
1807 
1808   def __init__(self, host, auth_function, host_override=None, extra_headers={},
1809                save_cookies=False):
1810     """Creates a new HttpRpcServer.
1811 
1812     Args:
1813       host: The host to send requests to.
1814       auth_function: A function that takes no arguments and returns an
1815         (email, password) tuple when called. Will be called if authentication
1816         is required.
1817       host_override: The host header to send to the server (defaults to host).
1818       extra_headers: A dict of extra headers to append to every request.
1819       save_cookies: If True, save the authentication cookies to local disk.
1820         If False, use an in-memory cookiejar instead.  Subclasses must
1821         implement this functionality.  Defaults to False.
1822     """
1823     self.host = host
1824     self.host_override = host_override
1825     self.auth_function = auth_function
1826     self.authenticated = False
1827     self.extra_headers = extra_headers
1828     self.save_cookies = save_cookies
1829     self.opener = self._GetOpener()
1830     if self.host_override:
1831       logging.info("Server: %s; Host: %s", self.host, self.host_override)
1832     else:
1833       logging.info("Server: %s", self.host)
1834 
1835   def _GetOpener(self):
1836     """Returns an OpenerDirector for making HTTP requests.
1837 
1838     Returns:
1839       A urllib2.OpenerDirector object.
1840     """
1841     raise NotImplementedError()
1842 
1843   def _CreateRequest(self, url, data=None):
1844     """Creates a new urllib request."""
1845     logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
1846     req = urllib2.Request(url, data=data)
1847     if self.host_override:
1848       req.add_header("Host", self.host_override)
1849     for key, value in self.extra_headers.iteritems():
1850       req.add_header(key, value)
1851     return req
1852 
1853   def _GetAuthToken(self, email, password):
1854     """Uses ClientLogin to authenticate the user, returning an auth token.
1855 
1856     Args:
1857       email:    The user's email address
1858       password: The user's password
1859 
1860     Raises:
1861       ClientLoginError: If there was an error authenticating with ClientLogin.
1862       HTTPError: If there was some other form of HTTP error.
1863 
1864     Returns:
1865       The authentication token returned by ClientLogin.
1866     """
1867     account_type = "GOOGLE"
1868     if self.host.endswith(".google.com") and not force_google_account:
1869       # Needed for use inside Google.
1870       account_type = "HOSTED"
1871     req = self._CreateRequest(
1872         url="https://www.google.com/accounts/ClientLogin",
1873         data=urllib.urlencode({
1874             "Email": email,
1875             "Passwd": password,
1876             "service": "ah",
1877             "source": "rietveld-codereview-upload",
1878             "accountType": account_type,
1879         }),
1880     )
1881     try:
1882       response = self.opener.open(req)
1883       response_body = response.read()
1884       response_dict = dict(x.split("=")
1885                            for x in response_body.split("\n") if x)
1886       return response_dict["Auth"]
1887     except urllib2.HTTPError, e:
1888       if e.code == 403:
1889         body = e.read()
1890         response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
1891         raise ClientLoginError(req.get_full_url(), e.code, e.msg,
1892                                e.headers, response_dict)
1893       else:
1894         raise
1895 
1896   def _GetAuthCookie(self, auth_token):
1897     """Fetches authentication cookies for an authentication token.
1898 
1899     Args:
1900       auth_token: The authentication token returned by ClientLogin.
1901 
1902     Raises:
1903       HTTPError: If there was an error fetching the authentication cookies.
1904     """
1905     # This is a dummy value to allow us to identify when we're successful.
1906     continue_location = "http://localhost/"
1907     args = {"continue": continue_location, "auth": auth_token}
1908     req = self._CreateRequest("http://%s/_ah/login?%s" %
1909                               (self.host, urllib.urlencode(args)))
1910     try:
1911       response = self.opener.open(req)
1912     except urllib2.HTTPError, e:
1913       response = e
1914     if (response.code != 302 or
1915         response.info()["location"] != continue_location):
1916       raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,
1917                               response.headers, response.fp)
1918     self.authenticated = True
1919 
1920   def _Authenticate(self):
1921     """Authenticates the user.
1922 
1923     The authentication process works as follows:
1924      1) We get a username and password from the user
1925      2) We use ClientLogin to obtain an AUTH token for the user
1926         (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
1927      3) We pass the auth token to /_ah/login on the server to obtain an
1928         authentication cookie. If login was successful, it tries to redirect
1929         us to the URL we provided.
1930 
1931     If we attempt to access the upload API without first obtaining an
1932     authentication cookie, it returns a 401 response (or a 302) and
1933     directs us to authenticate ourselves with ClientLogin.
1934     """
1935     for i in range(3):
1936       credentials = self.auth_function()
1937       try:
1938         auth_token = self._GetAuthToken(credentials[0], credentials[1])
1939       except ClientLoginError, e:
1940         if e.reason == "BadAuthentication":
1941           print >>sys.stderr, "Invalid username or password."
1942           continue
1943         if e.reason == "CaptchaRequired":
1944           print >>sys.stderr, (
1945               "Please go to\n"
1946               "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
1947               "and verify you are a human.  Then try again.")
1948           break
1949         if e.reason == "NotVerified":
1950           print >>sys.stderr, "Account not verified."
1951           break
1952         if e.reason == "TermsNotAgreed":
1953           print >>sys.stderr, "User has not agreed to TOS."
1954           break
1955         if e.reason == "AccountDeleted":
1956           print >>sys.stderr, "The user account has been deleted."
1957           break
1958         if e.reason == "AccountDisabled":
1959           print >>sys.stderr, "The user account has been disabled."
1960           break
1961         if e.reason == "ServiceDisabled":
1962           print >>sys.stderr, ("The user's access to the service has been "
1963                                "disabled.")
1964           break
1965         if e.reason == "ServiceUnavailable":
1966           print >>sys.stderr, "The service is not available; try again later."
1967           break
1968         raise
1969       self._GetAuthCookie(auth_token)
1970       return
1971 
1972   def Send(self, request_path, payload=None,
1973            content_type="application/octet-stream",
1974            timeout=None,
1975            **kwargs):
1976     """Sends an RPC and returns the response.
1977 
1978     Args:
1979       request_path: The path to send the request to, eg /api/appversion/create.
1980       payload: The body of the request, or None to send an empty request.
1981       content_type: The Content-Type header to use.
1982       timeout: timeout in seconds; default None i.e. no timeout.
1983         (Note: for large requests on OS X, the timeout doesn't work right.)
1984       kwargs: Any keyword arguments are converted into query string parameters.
1985 
1986     Returns:
1987       The response body, as a string.
1988     """
1989     # TODO: Don't require authentication.  Let the server say
1990     # whether it is necessary.
1991     if not self.authenticated:
1992       self._Authenticate()
1993 
1994     old_timeout = socket.getdefaulttimeout()
1995     socket.setdefaulttimeout(timeout)
1996     try:
1997       tries = 0
1998       while True:
1999         tries += 1
2000         args = dict(kwargs)
2001         url = "http://%s%s" % (self.host, request_path)
2002         if args:
2003           url += "?" + urllib.urlencode(args)
2004         req = self._CreateRequest(url=url, data=payload)
2005         req.add_header("Content-Type", content_type)
2006         try:
2007           f = self.opener.open(req)
2008           response = f.read()
2009           f.close()
2010           return response
2011         except urllib2.HTTPError, e:
2012           if tries > 3:
2013             raise
2014           elif e.code == 401 or e.code == 302:
2015             self._Authenticate()
2016           else:
2017             raise
2018     finally:
2019       socket.setdefaulttimeout(old_timeout)
2020 
2021 
2022 class HttpRpcServer(AbstractRpcServer):
2023   """Provides a simplified RPC-style interface for HTTP requests."""
2024 
2025   def _Authenticate(self):
2026     """Save the cookie jar after authentication."""
2027     super(HttpRpcServer, self)._Authenticate()
2028     if self.save_cookies:
2029       StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)
2030       self.cookie_jar.save()
2031 
2032   def _GetOpener(self):
2033     """Returns an OpenerDirector that supports cookies and ignores redirects.
2034 
2035     Returns:
2036       A urllib2.OpenerDirector object.
2037     """
2038     opener = urllib2.OpenerDirector()
2039     opener.add_handler(urllib2.ProxyHandler())
2040     opener.add_handler(urllib2.UnknownHandler())
2041     opener.add_handler(urllib2.HTTPHandler())
2042     opener.add_handler(urllib2.HTTPDefaultErrorHandler())
2043     opener.add_handler(urllib2.HTTPSHandler())
2044     opener.add_handler(urllib2.HTTPErrorProcessor())
2045     if self.save_cookies:
2046       self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies_" + server)
2047       self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
2048       if os.path.exists(self.cookie_file):
2049         try:
2050           self.cookie_jar.load()
2051           self.authenticated = True
2052           StatusUpdate("Loaded authentication cookies from %s" %
2053                        self.cookie_file)
2054         except (cookielib.LoadError, IOError):
2055           # Failed to load cookies - just ignore them.
2056           pass
2057       else:
2058         # Create an empty cookie file with mode 600
2059         fd = os.open(self.cookie_file, os.O_CREAT, 0600)
2060         os.close(fd)
2061       # Always chmod the cookie file
2062       os.chmod(self.cookie_file, 0600)
2063     else:
2064       # Don't save cookies across runs of update.py.
2065       self.cookie_jar = cookielib.CookieJar()
2066     opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
2067     return opener
2068 
2069 
2070 parser = optparse.OptionParser(usage="%prog [options] [-- diff_options]")
2071 parser.add_option("-y", "--assume_yes", action="store_true",
2072                   dest="assume_yes", default=False,
2073                   help="Assume that the answer to yes/no questions is 'yes'.")
2074 # Logging
2075 group = parser.add_option_group("Logging options")
2076 group.add_option("-q", "--quiet", action="store_const", const=0,
2077                  dest="verbose", help="Print errors only.")
2078 group.add_option("-v", "--verbose", action="store_const", const=2,
2079                  dest="verbose", default=1,
2080                  help="Print info level logs (default).")
2081 group.add_option("--noisy", action="store_const", const=3,
2082                  dest="verbose", help="Print all logs.")
2083 # Review server
2084 group = parser.add_option_group("Review server options")
2085 group.add_option("-s", "--server", action="store", dest="server",
2086                  default="codereview.appspot.com",
2087                  metavar="SERVER",
2088                  help=("The server to upload to. The format is host[:port]. "
2089                        "Defaults to '%default'."))
2090 group.add_option("-e", "--email", action="store", dest="email",
2091                  metavar="EMAIL", default=None,
2092                  help="The username to use. Will prompt if omitted.")
2093 group.add_option("-H", "--host", action="store", dest="host",
2094                  metavar="HOST", default=None,
2095                  help="Overrides the Host header sent with all RPCs.")
2096 group.add_option("--no_cookies", action="store_false",
2097                  dest="save_cookies", default=True,
2098                  help="Do not save authentication cookies to local disk.")
2099 # Issue
2100 group = parser.add_option_group("Issue options")
2101 group.add_option("-d", "--description", action="store", dest="description",
2102                  metavar="DESCRIPTION", default=None,
2103                  help="Optional description when creating an issue.")
2104 group.add_option("-f", "--description_file", action="store",
2105                  dest="description_file", metavar="DESCRIPTION_FILE",
2106                  default=None,
2107                  help="Optional path of a file that contains "
2108                       "the description when creating an issue.")
2109 group.add_option("-r", "--reviewers", action="store", dest="reviewers",
2110                  metavar="REVIEWERS", default=None,
2111                  help="Add reviewers (comma separated email addresses).")
2112 group.add_option("--cc", action="store", dest="cc",
2113                  metavar="CC", default=None,
2114                  help="Add CC (comma separated email addresses).")
2115 group.add_option("--private", action="store_true", dest="private",
2116                  default=False,
2117                  help="Make the issue restricted to reviewers and those CCed")
2118 # Upload options
2119 group = parser.add_option_group("Patch options")
2120 group.add_option("-m", "--message", action="store", dest="message",
2121                  metavar="MESSAGE", default=None,
2122                  help="A message to identify the patch. "
2123                       "Will prompt if omitted.")
2124 group.add_option("-i", "--issue", type="int", action="store",
2125                  metavar="ISSUE", default=None,
2126                  help="Issue number to which to add. Defaults to new issue.")
2127 group.add_option("--download_base", action="store_true",
2128                  dest="download_base", default=False,
2129                  help="Base files will be downloaded by the server "
2130                  "(side-by-side diffs may not work on files with CRs).")
2131 group.add_option("--rev", action="store", dest="revision",
2132                  metavar="REV", default=None,
2133                  help="Branch/tree/revision to diff against (used by DVCS).")
2134 group.add_option("--send_mail", action="store_true",
2135                  dest="send_mail", default=False,
2136                  help="Send notification email to reviewers.")
2137 group.add_option("--vcs", action="store", dest="vcs",
2138                  metavar="VCS", default=None,
2139                  help=("Version control system (optional, usually upload.py "
2140                        "already guesses the right VCS)."))
2141 
2142 
2143 def GetRpcServer(options):
2144   """Returns an instance of an AbstractRpcServer.
2145 
2146   Returns:
2147     A new AbstractRpcServer, on which RPC calls can be made.
2148   """
2149 
2150   rpc_server_class = HttpRpcServer
2151 
2152   def GetUserCredentials():
2153     """Prompts the user for a username and password."""
2154     email = options.email
2155     if email is None:
2156       email = GetEmail("Email (login for uploading to %s)" % options.server)
2157     password = getpass.getpass("Password for %s: " % email)
2158     return (email, password)
2159 
2160   # If this is the dev_appserver, use fake authentication.
2161   host = (options.host or options.server).lower()
2162   if host == "localhost" or host.startswith("localhost:"):
2163     email = options.email
2164     if email is None:
2165       email = "test@example.com"
2166       logging.info("Using debug user %s.  Override with --email" % email)
2167     server = rpc_server_class(
2168         options.server,
2169         lambda: (email, "password"),
2170         host_override=options.host,
2171         extra_headers={"Cookie":
2172                        'dev_appserver_login="%s:False"' % email},
2173         save_cookies=options.save_cookies)
2174     # Don't try to talk to ClientLogin.
2175     server.authenticated = True
2176     return server
2177 
2178   return rpc_server_class(options.server, GetUserCredentials,
2179                           host_override=options.host,
2180                           save_cookies=options.save_cookies)
2181 
2182 
2183 def EncodeMultipartFormData(fields, files):
2184   """Encode form fields for multipart/form-data.
2185 
2186   Args:
2187     fields: A sequence of (name, value) elements for regular form fields.
2188     files: A sequence of (name, filename, value) elements for data to be
2189            uploaded as files.
2190   Returns:
2191     (content_type, body) ready for httplib.HTTP instance.
2192 
2193   Source:
2194     http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
2195   """
2196   BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
2197   CRLF = '\r\n'
2198   lines = []
2199   for (key, value) in fields:
2200     lines.append('--' + BOUNDARY)
2201     lines.append('Content-Disposition: form-data; name="%s"' % key)
2202     lines.append('')
2203     if type(value) == unicode:
2204       value = value.encode("utf-8")
2205     lines.append(value)
2206   for (key, filename, value) in files:
2207     if type(filename) == unicode:
2208       filename = filename.encode("utf-8")
2209     if type(value) == unicode:
2210       value = value.encode("utf-8")
2211     lines.append('--' + BOUNDARY)
2212     lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
2213              (key, filename))
2214     lines.append('Content-Type: %s' % GetContentType(filename))
2215     lines.append('')
2216     lines.append(value)
2217   lines.append('--' + BOUNDARY + '--')
2218   lines.append('')
2219   body = CRLF.join(lines)
2220   content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
2221   return content_type, body
2222 
2223 
2224 def GetContentType(filename):
2225   """Helper to guess the content-type from the filename."""
2226   return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
2227 
2228 
2229 # Use a shell for subcommands on Windows to get a PATH search.
2230 use_shell = sys.platform.startswith("win")
2231 
2232 def RunShellWithReturnCode(command, print_output=False,
2233                            universal_newlines=True,
2234                            env=os.environ):
2235   """Executes a command and returns the output from stdout and the return code.
2236 
2237   Args:
2238     command: Command to execute.
2239     print_output: If True, the output is printed to stdout.
2240                   If False, both stdout and stderr are ignored.
2241     universal_newlines: Use universal_newlines flag (default: True).
2242 
2243   Returns:
2244     Tuple (output, return code)
2245   """
2246   logging.info("Running %s", command)
2247   p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2248                        shell=use_shell, universal_newlines=universal_newlines,
2249                        env=env)
2250   if print_output:
2251     output_array = []
2252     while True:
2253       line = p.stdout.readline()
2254       if not line:
2255         break
2256       print line.strip("\n")
2257       output_array.append(line)
2258     output = "".join(output_array)
2259   else:
2260     output = p.stdout.read()
2261   p.wait()
2262   errout = p.stderr.read()
2263   if print_output and errout:
2264     print >>sys.stderr, errout
2265   p.stdout.close()
2266   p.stderr.close()
2267   return output, p.returncode
2268 
2269 
2270 def RunShell(command, silent_ok=False, universal_newlines=True,
2271              print_output=False, env=os.environ):
2272   data, retcode = RunShellWithReturnCode(command, print_output,
2273                                          universal_newlines, env)
2274   if retcode:
2275     ErrorExit("Got error status from %s:\n%s" % (command, data))
2276   if not silent_ok and not data:
2277     ErrorExit("No output from %s" % command)
2278   return data
2279 
2280 
2281 class VersionControlSystem(object):
2282   """Abstract base class providing an interface to the VCS."""
2283 
2284   def __init__(self, options):
2285     """Constructor.
2286 
2287     Args:
2288       options: Command line options.
2289     """
2290     self.options = options
2291 
2292   def GenerateDiff(self, args):
2293     """Return the current diff as a string.
2294 
2295     Args:
2296       args: Extra arguments to pass to the diff command.
2297     """
2298     raise NotImplementedError(
2299         "abstract method -- subclass %s must override" % self.__class__)
2300 
2301   def GetUnknownFiles(self):
2302     """Return a list of files unknown to the VCS."""
2303     raise NotImplementedError(
2304         "abstract method -- subclass %s must override" % self.__class__)
2305 
2306   def CheckForUnknownFiles(self):
2307     """Show an "are you sure?" prompt if there are unknown files."""
2308     unknown_files = self.GetUnknownFiles()
2309     if unknown_files:
2310       print "The following files are not added to version control:"
2311       for line in unknown_files:
2312         print line
2313       prompt = "Are you sure to continue?(y/N) "
2314       answer = raw_input(prompt).strip()
2315       if answer != "y":
2316         ErrorExit("User aborted")
2317 
2318   def GetBaseFile(self, filename):
2319     """Get the content of the upstream version of a file.
2320 
2321     Returns:
2322       A tuple (base_content, new_content, is_binary, status)
2323         base_content: The contents of the base file.
2324         new_content: For text files, this is empty.  For binary files, this is
2325           the contents of the new file, since the diff output won't contain
2326           information to reconstruct the current file.
2327         is_binary: True iff the file is binary.
2328         status: The status of the file.
2329     """
2330 
2331     raise NotImplementedError(
2332         "abstract method -- subclass %s must override" % self.__class__)
2333 
2334 
2335   def GetBaseFiles(self, diff):
2336     """Helper that calls GetBase file for each file in the patch.
2337 
2338     Returns:
2339       A dictionary that maps from filename to GetBaseFile's tuple.  Filenames
2340       are retrieved based on lines that start with "Index:" or
2341       "Property changes on:".
2342     """
2343     files = {}
2344     for line in diff.splitlines(True):
2345       if line.startswith('Index:') or line.startswith('Property changes on:'):
2346         unused, filename = line.split(':', 1)
2347         # On Windows if a file has property changes its filename uses '\'
2348         # instead of '/'.
2349         filename = filename.strip().replace('\\', '/')
2350         files[filename] = self.GetBaseFile(filename)
2351     return files
2352 
2353 
2354   def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options,
2355                       files):
2356     """Uploads the base files (and if necessary, the current ones as well)."""
2357 
2358     def UploadFile(filename, file_id, content, is_binary, status, is_base):
2359       """Uploads a file to the server."""
2360       file_too_large = False
2361       if is_base:
2362         type = "base"
2363       else:
2364         type = "current"
2365       if len(content) > MAX_UPLOAD_SIZE:
2366         print ("Not uploading the %s file for %s because it's too large." %
2367                (type, filename))
2368         file_too_large = True
2369         content = ""
2370       checksum = md5(content).hexdigest()
2371       if options.verbose > 0 and not file_too_large:
2372         print "Uploading %s file for %s" % (type, filename)
2373       url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id)
2374       form_fields = [("filename", filename),
2375                      ("status", status),
2376                      ("checksum", checksum),
2377                      ("is_binary", str(is_binary)),
2378                      ("is_current", str(not is_base)),
2379                     ]
2380       if file_too_large:
2381         form_fields.append(("file_too_large", "1"))
2382       if options.email:
2383         form_fields.append(("user", options.email))
2384       ctype, body = EncodeMultipartFormData(form_fields,
2385                                             [("data", filename, content)])
2386       response_body = rpc_server.Send(url, body,
2387                                       content_type=ctype)
2388       if not response_body.startswith("OK"):
2389         StatusUpdate("  --> %s" % response_body)
2390         sys.exit(1)
2391 
2392     patches = dict()
2393     [patches.setdefault(v, k) for k, v in patch_list]
2394     for filename in patches.keys():
2395       base_content, new_content, is_binary, status = files[filename]
2396       file_id_str = patches.get(filename)
2397       if file_id_str.find("nobase") != -1:
2398         base_content = None
2399         file_id_str = file_id_str[file_id_str.rfind("_") + 1:]
2400       file_id = int(file_id_str)
2401       if base_content != None:
2402         UploadFile(filename, file_id, base_content, is_binary, status, True)
2403       if new_content != None:
2404         UploadFile(filename, file_id, new_content, is_binary, status, False)
2405 
2406   def IsImage(self, filename):
2407     """Returns true if the filename has an image extension."""
2408     mimetype =  mimetypes.guess_type(filename)[0]
2409     if not mimetype:
2410       return False
2411     return mimetype.startswith("image/")
2412 
2413   def IsBinary(self, filename):
2414     """Returns true if the guessed mimetyped isnt't in text group."""
2415     mimetype = mimetypes.guess_type(filename)[0]
2416     if not mimetype:
2417       return False  # e.g. README, "real" binaries usually have an extension
2418     # special case for text files which don't start with text/
2419     if mimetype in TEXT_MIMETYPES:
2420       return False
2421     return not mimetype.startswith("text/")
2422 
2423 
2424 class SubversionVCS(VersionControlSystem):
2425   """Implementation of the VersionControlSystem interface for Subversion."""
2426 
2427   def __init__(self, options):
2428     super(SubversionVCS, self).__init__(options)
2429     if self.options.revision:
2430       match = re.match(r"(\d+)(:(\d+))?", self.options.revision)
2431       if not match:
2432         ErrorExit("Invalid Subversion revision %s." % self.options.revision)
2433       self.rev_start = match.group(1)
2434       self.rev_end = match.group(3)
2435     else:
2436       self.rev_start = self.rev_end = None
2437     # Cache output from "svn list -r REVNO dirname".
2438     # Keys: dirname, Values: 2-tuple (ouput for start rev and end rev).
2439     self.svnls_cache = {}
2440     # SVN base URL is required to fetch files deleted in an older revision.
2441     # Result is cached to not guess it over and over again in GetBaseFile().
2442     required = self.options.download_base or self.options.revision is not None
2443     self.svn_base = self._GuessBase(required)
2444 
2445   def GuessBase(self, required):
2446     """Wrapper for _GuessBase."""
2447     return self.svn_base
2448 
2449   def _GuessBase(self, required):
2450     """Returns the SVN base URL.
2451 
2452     Args:
2453       required: If true, exits if the url can't be guessed, otherwise None is
2454         returned.
2455     """
2456     info = RunShell(["svn", "info"])
2457     for line in info.splitlines():
2458       words = line.split()
2459       if len(words) == 2 and words[0] == "URL:":
2460         url = words[1]
2461         scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
2462         username, netloc = urllib.splituser(netloc)
2463         if username:
2464           logging.info("Removed username from base URL")
2465         if netloc.endswith("svn.python.org"):
2466           if netloc == "svn.python.org":
2467             if path.startswith("/projects/"):
2468               path = path[9:]
2469           elif netloc != "pythondev@svn.python.org":
2470             ErrorExit("Unrecognized Python URL: %s" % url)
2471           base = "http://svn.python.org/view/*checkout*%s/" % path
2472           logging.info("Guessed Python base = %s", base)
2473         elif netloc.endswith("svn.collab.net"):
2474           if path.startswith("/repos/"):
2475             path = path[6:]
2476           base = "http://svn.collab.net/viewvc/*checkout*%s/" % path
2477           logging.info("Guessed CollabNet base = %s", base)
2478         elif netloc.endswith(".googlecode.com"):
2479           path = path + "/"
2480           base = urlparse.urlunparse(("http", netloc, path, params,
2481                                       query, fragment))
2482           logging.info("Guessed Google Code base = %s", base)
2483         else:
2484           path = path + "/"
2485           base = urlparse.urlunparse((scheme, netloc, path, params,
2486                                       query, fragment))
2487           logging.info("Guessed base = %s", base)
2488         return base
2489     if required:
2490       ErrorExit("Can't find URL in output from svn info")
2491     return None
2492 
2493   def GenerateDiff(self, args):
2494     cmd = ["svn", "diff"]
2495     if self.options.revision:
2496       cmd += ["-r", self.options.revision]
2497     cmd.extend(args)
2498     data = RunShell(cmd)
2499     count = 0
2500     for line in data.splitlines():
2501       if line.startswith("Index:") or line.startswith("Property changes on:"):
2502         count += 1
2503         logging.info(line)
2504     if not count:
2505       ErrorExit("No valid patches found in output from svn diff")
2506     return data
2507 
2508   def _CollapseKeywords(self, content, keyword_str):
2509     """Collapses SVN keywords."""
2510     # svn cat translates keywords but svn diff doesn't. As a result of this
2511     # behavior patching.PatchChunks() fails with a chunk mismatch error.
2512     # This part was originally written by the Review Board development team
2513     # who had the same problem (http://reviews.review-board.org/r/276/).
2514     # Mapping of keywords to known aliases
2515     svn_keywords = {
2516       # Standard keywords
2517       'Date':                ['Date', 'LastChangedDate'],
2518       'Revision':            ['Revision', 'LastChangedRevision', 'Rev'],
2519       'Author':              ['Author', 'LastChangedBy'],
2520       'HeadURL':             ['HeadURL', 'URL'],
2521       'Id':                  ['Id'],
2522 
2523       # Aliases
2524       'LastChangedDate':     ['LastChangedDate', 'Date'],
2525       'LastChangedRevision': ['LastChangedRevision', 'Rev', 'Revision'],
2526       'LastChangedBy':       ['LastChangedBy', 'Author'],
2527       'URL':                 ['URL', 'HeadURL'],
2528     }
2529 
2530     def repl(m):
2531        if m.group(2):
2532          return "$%s::%s$" % (m.group(1), " " * len(m.group(3)))
2533        return "$%s$" % m.group(1)
2534     keywords = [keyword
2535                 for name in keyword_str.split(" ")
2536                 for keyword in svn_keywords.get(name, [])]
2537     return re.sub(r"\$(%s):(:?)([^\$]+)\$" % '|'.join(keywords), repl, content)
2538 
2539   def GetUnknownFiles(self):
2540     status = RunShell(["svn", "status", "--ignore-externals"], silent_ok=True)
2541     unknown_files = []
2542     for line in status.split("\n"):
2543       if line and line[0] == "?":
2544         unknown_files.append(line)
2545     return unknown_files
2546 
2547   def ReadFile(self, filename):
2548     """Returns the contents of a file."""
2549     file = open(filename, 'rb')
2550     result = ""
2551     try:
2552       result = file.read()
2553     finally:
2554       file.close()
2555     return result
2556 
2557   def GetStatus(self, filename):
2558     """Returns the status of a file."""
2559     if not self.options.revision:
2560       status = RunShell(["svn", "status", "--ignore-externals", filename])
2561       if not status:
2562         ErrorExit("svn status returned no output for %s" % filename)
2563       status_lines = status.splitlines()
2564       # If file is in a cl, the output will begin with
2565       # "\n--- Changelist 'cl_name':\n".  See
2566       # http://svn.collab.net/repos/svn/trunk/notes/changelist-design.txt
2567       if (len(status_lines) == 3 and
2568           not status_lines[0] and
2569           status_lines[1].startswith("--- Changelist")):
2570         status = status_lines[2]
2571       else:
2572         status = status_lines[0]
2573     # If we have a revision to diff against we need to run "svn list"
2574     # for the old and the new revision and compare the results to get
2575     # the correct status for a file.
2576     else:
2577       dirname, relfilename = os.path.split(filename)
2578       if dirname not in self.svnls_cache:
2579         cmd = ["svn", "list", "-r", self.rev_start, dirname or "."]
2580         out, returncode = RunShellWithReturnCode(cmd)
2581         if returncode:
2582           ErrorExit("Failed to get status for %s." % filename)
2583         old_files = out.splitlines()
2584         args = ["svn", "list"]
2585         if self.rev_end:
2586           args += ["-r", self.rev_end]
2587         cmd = args + [dirname or "."]
2588         out, returncode = RunShellWithReturnCode(cmd)
2589         if returncode:
2590           ErrorExit("Failed to run command %s" % cmd)
2591         self.svnls_cache[dirname] = (old_files, out.splitlines())
2592       old_files, new_files = self.svnls_cache[dirname]
2593       if relfilename in old_files and relfilename not in new_files:
2594         status = "D   "
2595       elif relfilename in old_files and relfilename in new_files:
2596         status = "M   "
2597       else:
2598         status = "A   "
2599     return status
2600 
2601   def GetBaseFile(self, filename):
2602     status = self.GetStatus(filename)
2603     base_content = None
2604     new_content = None
2605 
2606     # If a file is copied its status will be "A  +", which signifies
2607     # "addition-with-history".  See "svn st" for more information.  We need to
2608     # upload the original file or else diff parsing will fail if the file was
2609     # edited.
2610     if status[0] == "A" and status[3] != "+":
2611       # We'll need to upload the new content if we're adding a binary file
2612       # since diff's output won't contain it.
2613       mimetype = RunShell(["svn", "propget", "svn:mime-type", filename],
2614                           silent_ok=True)
2615       base_content = ""
2616       is_binary = bool(mimetype) and not mimetype.startswith("text/")
2617       if is_binary and self.IsImage(filename):
2618         new_content = self.ReadFile(filename)
2619     elif (status[0] in ("M", "D", "R") or
2620           (status[0] == "A" and status[3] == "+") or  # Copied file.
2621           (status[0] == " " and status[1] == "M")):  # Property change.
2622       args = []
2623       if self.options.revision:
2624         url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
2625       else:
2626         # Don't change filename, it's needed later.
2627         url = filename
2628         args += ["-r", "BASE"]
2629       cmd = ["svn"] + args + ["propget", "svn:mime-type", url]
2630       mimetype, returncode = RunShellWithReturnCode(cmd)
2631       if returncode:
2632         # File does not exist in the requested revision.
2633         # Reset mimetype, it contains an error message.
2634         mimetype = ""
2635       get_base = False
2636       is_binary = bool(mimetype) and not mimetype.startswith("text/")
2637       if status[0] == " ":
2638         # Empty base content just to force an upload.
2639         base_content = ""
2640       elif is_binary:
2641         if self.IsImage(filename):
2642           get_base = True
2643           if status[0] == "M":
2644             if not self.rev_end:
2645               new_content = self.ReadFile(filename)
2646             else:
2647               url = "%s/%s@%s" % (self.svn_base, filename, self.rev_end)
2648               new_content = RunShell(["svn", "cat", url],
2649                                      universal_newlines=True, silent_ok=True)
2650         else:
2651           base_content = ""
2652       else:
2653         get_base = True
2654 
2655       if get_base:
2656         if is_binary:
2657           universal_newlines = False
2658         else:
2659           universal_newlines = True
2660         if self.rev_start:
2661           # "svn cat -r REV delete_file.txt" doesn't work. cat requires
2662           # the full URL with "@REV" appended instead of using "-r" option.
2663           url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
2664           base_content = RunShell(["svn", "cat", url],
2665                                   universal_newlines=universal_newlines,
2666                                   silent_ok=True)
2667         else:
2668           base_content = RunShell(["svn", "cat", filename],
2669                                   universal_newlines=universal_newlines,
2670                                   silent_ok=True)
2671         if not is_binary:
2672           args = []
2673           if self.rev_start:
2674             url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
2675           else:
2676             url = filename
2677             args += ["-r", "BASE"]
2678           cmd = ["svn"] + args + ["propget", "svn:keywords", url]
2679           keywords, returncode = RunShellWithReturnCode(cmd)
2680           if keywords and not returncode:
2681             base_content = self._CollapseKeywords(base_content, keywords)
2682     else:
2683       StatusUpdate("svn status returned unexpected output: %s" % status)
2684       sys.exit(1)
2685     return base_content, new_content, is_binary, status[0:5]
2686 
2687 
2688 class GitVCS(VersionControlSystem):
2689   """Implementation of the VersionControlSystem interface for Git."""
2690 
2691   def __init__(self, options):
2692     super(GitVCS, self).__init__(options)
2693     # Map of filename -> (hash before, hash after) of base file.
2694     # Hashes for "no such file" are represented as None.
2695     self.hashes = {}
2696     # Map of new filename -> old filename for renames.
2697     self.renames = {}
2698 
2699   def GenerateDiff(self, extra_args):
2700     # This is more complicated than svn's GenerateDiff because we must convert
2701     # the diff output to include an svn-style "Index:" line as well as record
2702     # the hashes of the files, so we can upload them along with our diff.
2703 
2704     # Special used by git to indicate "no such content".
2705     NULL_HASH = "0"*40
2706 
2707     extra_args = extra_args[:]
2708     if self.options.revision:
2709       extra_args = [self.options.revision] + extra_args
2710     extra_args.append('-M')
2711 
2712     # --no-ext-diff is broken in some versions of Git, so try to work around
2713     # this by overriding the environment (but there is still a problem if the
2714     # git config key "diff.external" is used).
2715     env = os.environ.copy()
2716     if 'GIT_EXTERNAL_DIFF' in env: del env['GIT_EXTERNAL_DIFF']
2717     gitdiff = RunShell(["git", "diff", "--no-ext-diff", "--full-index"]
2718                        + extra_args, env=env)
2719     svndiff = []
2720     filecount = 0
2721     filename = None
2722     for line in gitdiff.splitlines():
2723       match = re.match(r"diff --git a/(.*) b/(.*)$", line)
2724       if match:
2725         filecount += 1
2726         # Intentionally use the "after" filename so we can show renames.
2727         filename = match.group(2)
2728         svndiff.append("Index: %s\n" % filename)
2729         if match.group(1) != match.group(2):
2730           self.renames[match.group(2)] = match.group(1)
2731       else:
2732         # The "index" line in a git diff looks like this (long hashes elided):
2733         #   index 82c0d44..b2cee3f 100755
2734         # We want to save the left hash, as that identifies the base file.
2735         match = re.match(r"index (\w+)\.\.(\w+)", line)
2736         if match:
2737           before, after = (match.group(1), match.group(2))
2738           if before == NULL_HASH:
2739             before = None
2740           if after == NULL_HASH:
2741             after = None
2742           self.hashes[filename] = (before, after)
2743       svndiff.append(line + "\n")
2744     if not filecount:
2745       ErrorExit("No valid patches found in output from git diff")
2746     return "".join(svndiff)
2747 
2748   def GetUnknownFiles(self):
2749     status = RunShell(["git", "ls-files", "--exclude-standard", "--others"],
2750                       silent_ok=True)
2751     return status.splitlines()
2752 
2753   def GetFileContent(self, file_hash, is_binary):
2754     """Returns the content of a file identified by its git hash."""
2755     data, retcode = RunShellWithReturnCode(["git", "show", file_hash],
2756                                             universal_newlines=not is_binary)
2757     if retcode:
2758       ErrorExit("Got error status from 'git show %s'" % file_hash)
2759     return data
2760 
2761   def GetBaseFile(self, filename):
2762     hash_before, hash_after = self.hashes.get(filename, (None,None))
2763     base_content = None
2764     new_content = None
2765     is_binary = self.IsBinary(filename)
2766     status = None
2767 
2768     if filename in self.renames:
2769       status = "A +"  # Match svn attribute name for renames.
2770       if filename not in self.hashes:
2771         # If a rename doesn't change the content, we never get a hash.
2772         base_content = RunShell(["git", "show", filename])
2773     elif not hash_before:
2774       status = "A"
2775       base_content = ""
2776     elif not hash_after:
2777       status = "D"
2778     else:
2779       status = "M"
2780 
2781     is_image = self.IsImage(filename)
2782 
2783     # Grab the before/after content if we need it.
2784     # We should include file contents if it's text or it's an image.
2785     if not is_binary or is_image:
2786       # Grab the base content if we don't have it already.
2787       if base_content is None and hash_before:
2788         base_content = self.GetFileContent(hash_before, is_binary)
2789       # Only include the "after" file if it's an image; otherwise it
2790       # it is reconstructed from the diff.
2791       if is_image and hash_after:
2792         new_content = self.GetFileContent(hash_after, is_binary)
2793 
2794     return (base_content, new_content, is_binary, status)
2795 
2796 
2797 class MercurialVCS(VersionControlSystem):
2798   """Implementation of the VersionControlSystem interface for Mercurial."""
2799 
2800   def __init__(self, options, repo_dir):
2801     super(MercurialVCS, self).__init__(options)
2802     # Absolute path to repository (we can be in a subdir)
2803     self.repo_dir = os.path.normpath(repo_dir)
2804     # Compute the subdir
2805     cwd = os.path.normpath(os.getcwd())
2806     assert cwd.startswith(self.repo_dir)
2807     self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/")
2808     if self.options.revision:
2809       self.base_rev = self.options.revision
2810     else:
2811       self.base_rev = RunShell(["hg", "parent", "-q"]).split(':')[1].strip()
2812 
2813   def _GetRelPath(self, filename):
2814     """Get relative path of a file according to the current directory,
2815     given its logical path in the repo."""
2816     assert filename.startswith(self.subdir), (filename, self.subdir)
2817     return filename[len(self.subdir):].lstrip(r"\/")
2818 
2819   def GenerateDiff(self, extra_args):
2820     # If no file specified, restrict to the current subdir
2821     extra_args = extra_args or ["."]
2822     cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
2823     data = RunShell(cmd, silent_ok=True)
2824     svndiff = []
2825     filecount = 0
2826     for line in data.splitlines():
2827       m = re.match("diff --git a/(\S+) b/(\S+)", line)
2828       if m:
2829         # Modify line to make it look like as it comes from svn diff.
2830         # With this modification no changes on the server side are required
2831         # to make upload.py work with Mercurial repos.
2832         # NOTE: for proper handling of moved/copied files, we have to use
2833         # the second filename.
2834         filename = m.group(2)
2835         svndiff.append("Index: %s" % filename)
2836         svndiff.append("=" * 67)
2837         filecount += 1
2838         logging.info(line)
2839       else:
2840         svndiff.append(line)
2841     if not filecount:
2842       ErrorExit("No valid patches found in output from hg diff")
2843     return "\n".join(svndiff) + "\n"
2844 
2845   def GetUnknownFiles(self):
2846     """Return a list of files unknown to the VCS."""
2847     args = []
2848     status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
2849         silent_ok=True)
2850     unknown_files = []
2851     for line in status.splitlines():
2852       st, fn = line.split(" ", 1)
2853       if st == "?":
2854         unknown_files.append(fn)
2855     return unknown_files
2856 
2857   def GetBaseFile(self, filename):
2858     # "hg status" and "hg cat" both take a path relative to the current subdir
2859     # rather than to the repo root, but "hg diff" has given us the full path
2860     # to the repo root.
2861     base_content = ""
2862     new_content = None
2863     is_binary = False
2864     oldrelpath = relpath = self._GetRelPath(filename)
2865     # "hg status -C" returns two lines for moved/copied files, one otherwise
2866     out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath])
2867     out = out.splitlines()
2868     # HACK: strip error message about missing file/directory if it isn't in
2869     # the working copy
2870     if out[0].startswith('%s: ' % relpath):
2871       out = out[1:]
2872     if len(out) > 1:
2873       # Moved/copied => considered as modified, use old filename to
2874       # retrieve base contents
2875       oldrelpath = out[1].strip()
2876       status = "M"
2877     else:
2878       status, _ = out[0].split(' ', 1)
2879     if ":" in self.base_rev:
2880       base_rev = self.base_rev.split(":", 1)[0]
2881     else:
2882       base_rev = self.base_rev
2883     if status != "A":
2884       base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
2885         silent_ok=True)
2886       is_binary = "\0" in base_content  # Mercurial's heuristic
2887     if status != "R":
2888       new_content = open(relpath, "rb").read()
2889       is_binary = is_binary or "\0" in new_content
2890     if is_binary and base_content:
2891       # Fetch again without converting newlines
2892       base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
2893         silent_ok=True, universal_newlines=False)
2894     if not is_binary or not self.IsImage(relpath):
2895       new_content = None
2896     return base_content, new_content, is_binary, status
2897 
2898 
2899 # NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
2900 def SplitPatch(data):
2901   """Splits a patch into separate pieces for each file.
2902 
2903   Args:
2904     data: A string containing the output of svn diff.
2905 
2906   Returns:
2907     A list of 2-tuple (filename, text) where text is the svn diff output
2908       pertaining to filename.
2909   """
2910   patches = []
2911   filename = None
2912   diff = []
2913   for line in data.splitlines(True):
2914     new_filename = None
2915     if line.startswith('Index:'):
2916       unused, new_filename = line.split(':', 1)
2917       new_filename = new_filename.strip()
2918     elif line.startswith('Property changes on:'):
2919       unused, temp_filename = line.split(':', 1)
2920       # When a file is modified, paths use '/' between directories, however
2921       # when a property is modified '\' is used on Windows.  Make them the same
2922       # otherwise the file shows up twice.
2923       temp_filename = temp_filename.strip().replace('\\', '/')
2924       if temp_filename != filename:
2925         # File has property changes but no modifications, create a new diff.
2926         new_filename = temp_filename
2927     if new_filename:
2928       if filename and diff:
2929         patches.append((filename, ''.join(diff)))
2930       filename = new_filename
2931       diff = [line]
2932       continue
2933     if diff is not None:
2934       diff.append(line)
2935   if filename and diff:
2936     patches.append((filename, ''.join(diff)))
2937   return patches
2938 
2939 
2940 def UploadSeparatePatches(issue, rpc_server, patchset, data, options):
2941   """Uploads a separate patch for each file in the diff output.
2942 
2943   Returns a list of [patch_key, filename] for each file.
2944   """
2945   patches = SplitPatch(data)
2946   rv = []
2947   for patch in patches:
2948     if len(patch[1]) > MAX_UPLOAD_SIZE:
2949       print ("Not uploading the patch for " + patch[0] +
2950              " because the file is too large.")
2951       continue
2952     form_fields = [("filename", patch[0])]
2953     if not options.download_base:
2954       form_fields.append(("content_upload", "1"))
2955     files = [("data", "data.diff", patch[1])]
2956     ctype, body = EncodeMultipartFormData(form_fields, files)
2957     url = "/%d/upload_patch/%d" % (int(issue), int(patchset))
2958     print "Uploading patch for " + patch[0]
2959     response_body = rpc_server.Send(url, body, content_type=ctype)
2960     lines = response_body.splitlines()
2961     if not lines or lines[0] != "OK":
2962       StatusUpdate("  --> %s" % response_body)
2963       sys.exit(1)
2964     rv.append([lines[1], patch[0]])
2965   return rv
2966 
2967 
2968 def GuessVCSName():
2969   """Helper to guess the version control system.
2970 
2971   This examines the current directory, guesses which VersionControlSystem
2972   we're using, and returns an string indicating which VCS is detected.
2973 
2974   Returns:
2975     A pair (vcs, output).  vcs is a string indicating which VCS was detected
2976     and is one of VCS_GIT, VCS_MERCURIAL, VCS_SUBVERSION, or VCS_UNKNOWN.
2977     output is a string containing any interesting output from the vcs
2978     detection routine, or None if there is nothing interesting.
2979   """
2980   # Mercurial has a command to get the base directory of a repository
2981   # Try running it, but don't die if we don't have hg installed.
2982   # NOTE: we try Mercurial first as it can sit on top of an SVN working copy.
2983   try:
2984     out, returncode = RunShellWithReturnCode(["hg", "root"])
2985     if returncode == 0:
2986       return (VCS_MERCURIAL, out.strip())
2987   except OSError, (errno, message):
2988     if errno != 2:  # ENOENT -- they don't have hg installed.
2989       raise
2990 
2991   # Subversion has a .svn in all working directories.
2992   if os.path.isdir('.svn'):
2993     logging.info("Guessed VCS = Subversion")
2994     return (VCS_SUBVERSION, None)
2995 
2996   # Git has a command to test if you're in a git tree.
2997   # Try running it, but don't die if we don't have git installed.
2998   try:
2999     out, returncode = RunShellWithReturnCode(["git", "rev-parse",
3000                                               "--is-inside-work-tree"])
3001     if returncode == 0:
3002       return (VCS_GIT, None)
3003   except OSError, (errno, message):
3004     if errno != 2:  # ENOENT -- they don't have git installed.
3005       raise
3006 
3007   return (VCS_UNKNOWN, None)
3008 
3009 
3010 def GuessVCS(options):
3011   """Helper to guess the version control system.
3012 
3013   This verifies any user-specified VersionControlSystem (by command line
3014   or environment variable).  If the user didn't specify one, this examines
3015   the current directory, guesses which VersionControlSystem we're using,
3016   and returns an instance of the appropriate class.  Exit with an error
3017   if we can't figure it out.
3018 
3019   Returns:
3020     A VersionControlSystem instance. Exits if the VCS can't be guessed.
3021   """
3022   vcs = options.vcs
3023   if not vcs:
3024     vcs = os.environ.get("CODEREVIEW_VCS")
3025   if vcs:
3026     v = VCS_ABBREVIATIONS.get(vcs.lower())
3027     if v is None:
3028       ErrorExit("Unknown version control system %r specified." % vcs)
3029     (vcs, extra_output) = (v, None)
3030   else:
3031     (vcs, extra_output) = GuessVCSName()
3032 
3033   if vcs == VCS_MERCURIAL:
3034     if extra_output is None:
3035       extra_output = RunShell(["hg", "root"]).strip()
3036     return MercurialVCS(options, extra_output)
3037   elif vcs == VCS_SUBVERSION:
3038     return SubversionVCS(options)
3039   elif vcs == VCS_GIT:
3040     return GitVCS(options)
3041 
3042   ErrorExit(("Could not guess version control system. "
3043              "Are you in a working copy directory?"))
3044 
3045 
3046 def RealMain(argv, data=None):
3047   """The real main function.
3048 
3049   Args:
3050     argv: Command line arguments.
3051     data: Diff contents. If None (default) the diff is generated by
3052       the VersionControlSystem implementation returned by GuessVCS().
3053 
3054   Returns:
3055     A 2-tuple (issue id, patchset id).
3056     The patchset id is None if the base files are not uploaded by this
3057     script (applies only to SVN checkouts).
3058   """
3059   logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:"
3060                               "%(lineno)s %(message)s "))
3061   os.environ['LC_ALL'] = 'C'
3062   options, args = parser.parse_args(argv[1:])
3063   global verbosity
3064   verbosity = options.verbose
3065   if verbosity >= 3:
3066     logging.getLogger().setLevel(logging.DEBUG)
3067   elif verbosity >= 2:
3068     logging.getLogger().setLevel(logging.INFO)
3069   vcs = GuessVCS(options)
3070   if isinstance(vcs, SubversionVCS):
3071     # base field is only allowed for Subversion.
3072     # Note: Fetching base files may become deprecated in future releases.
3073     base = vcs.GuessBase(options.download_base)
3074   else:
3075     base = None
3076   if not base and options.download_base:
3077     options.download_base = True
3078     logging.info("Enabled upload of base file")
3079   if not options.assume_yes:
3080     vcs.CheckForUnknownFiles()
3081   if data is None:
3082     data = vcs.GenerateDiff(args)
3083   files = vcs.GetBaseFiles(data)
3084   if verbosity >= 1:
3085     print "Upload server:", options.server, "(change with -s/--server)"
3086   if options.issue:
3087     prompt = "Message describing this patch set: "
3088   else:
3089     prompt = "New issue subject: "
3090   message = options.message or raw_input(prompt).strip()
3091   if not message:
3092     ErrorExit("A non-empty message is required")
3093   rpc_server = GetRpcServer(options)
3094   form_fields = [("subject", message)]
3095   if base:
3096     form_fields.append(("base", base))
3097   if options.issue:
3098     form_fields.append(("issue", str(options.issue)))
3099   if options.email:
3100     form_fields.append(("user", options.email))
3101   if options.reviewers:
3102     for reviewer in options.reviewers.split(','):
3103       if "@" in reviewer and not reviewer.split("@")[1].count(".") == 1:
3104         ErrorExit("Invalid email address: %s" % reviewer)
3105     form_fields.append(("reviewers", options.reviewers))
3106   if options.cc:
3107     for cc in options.cc.split(','):
3108       if "@" in cc and not cc.split("@")[1].count(".") == 1:
3109         ErrorExit("Invalid email address: %s" % cc)
3110     form_fields.append(("cc", options.cc))
3111   description = options.description
3112   if options.description_file:
3113     if options.description:
3114       ErrorExit("Can't specify description and description_file")
3115     file = open(options.description_file, 'r')
3116     description = file.read()
3117     file.close()
3118   if description:
3119     form_fields.append(("description", description))
3120   # Send a hash of all the base file so the server can determine if a copy
3121   # already exists in an earlier patchset.
3122   base_hashes = ""
3123   for file, info in files.iteritems():
3124     if not info[0] is None:
3125       checksum = md5(info[0]).hexdigest()
3126       if base_hashes:
3127         base_hashes += "|"
3128       base_hashes += checksum + ":" + file
3129   form_fields.append(("base_hashes", base_hashes))
3130   if options.private:
3131     if options.issue:
3132       print "Warning: Private flag ignored when updating an existing issue."
3133     else:
3134       form_fields.append(("private", "1"))
3135   # If we're uploading base files, don't send the email before the uploads, so
3136   # that it contains the file status.
3137   if options.send_mail and options.download_base:
3138     form_fields.append(("send_mail", "1"))
3139   if not options.download_base:
3140     form_fields.append(("content_upload", "1"))
3141   if len(data) > MAX_UPLOAD_SIZE:
3142     print "Patch is large, so uploading file patches separately."
3143     uploaded_diff_file = []
3144     form_fields.append(("separate_patches", "1"))
3145   else:
3146     uploaded_diff_file = [("data", "data.diff", data)]
3147   ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
3148   response_body = rpc_server.Send("/upload", body, content_type=ctype)
3149   patchset = None
3150   if not options.download_base or not uploaded_diff_file:
3151     lines = response_body.splitlines()
3152     if len(lines) >= 2:
3153       msg = lines[0]
3154       patchset = lines[1].strip()
3155       patches = [x.split(" ", 1) for x in lines[2:]]
3156     else:
3157       msg = response_body
3158   else:
3159     msg = response_body
3160   if not response_body.startswith("Issue created.") and \
3161   not response_body.startswith("Issue updated."):
3162     print >>sys.stderr, msg
3163     sys.exit(0)
3164   issue = msg[msg.rfind("/")+1:]
3165 
3166   if not uploaded_diff_file:
3167     result = UploadSeparatePatches(issue, rpc_server, patchset, data, options)
3168     if not options.download_base:
3169       patches = result
3170 
3171   if not options.download_base:
3172     vcs.UploadBaseFiles(issue, rpc_server, patches, patchset, options, files)
3173     if options.send_mail:
3174       rpc_server.Send("/" + issue + "/mail", payload="")
3175   return issue, patchset
3176 
3177 
3178 def main():
3179   try:
3180     RealMain(sys.argv)
3181   except KeyboardInterrupt:
3182     print
3183     StatusUpdate("Interrupted.")
3184     sys.exit(1)
3185